KMOS Pipeline Reference Manual  1.2.0
kmo_illumination.c
00001 /* $Id: kmo_illumination.c,v 1.55 2013/06/17 07:52:26 aagudo Exp $
00002  *
00003  * This file is part of the KMOS Pipeline
00004  * Copyright (C) 2002,2003 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: aagudo $
00023  * $Date: 2013/06/17 07:52:26 $
00024  * $Revision: 1.55 $
00025  * $Name: HEAD $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 /*-----------------------------------------------------------------------------
00033  *                              Includes
00034  *----------------------------------------------------------------------------*/
00035 
00036 #include <math.h>
00037 #include <string.h>
00038 
00039 #include <cpl.h>
00040 
00041 #include "kmo_priv_reconstruct.h"
00042 #include "kmo_priv_functions.h"
00043 #include "kmo_priv_flat.h"
00044 #include "kmo_priv_wave_cal.h"
00045 #include "kmclipm_priv_splines.h"
00046 #include "kmo_functions.h"
00047 #include "kmo_cpl_extensions.h"
00048 #include "kmo_dfs.h"
00049 #include "kmo_error.h"
00050 #include "kmo_constants.h"
00051 #include "kmo_debug.h"
00052 
00053 /*-----------------------------------------------------------------------------
00054  *                          Functions prototypes
00055  *----------------------------------------------------------------------------*/
00056 
00057 static int kmo_illumination_create(cpl_plugin *);
00058 static int kmo_illumination_exec(cpl_plugin *);
00059 static int kmo_illumination_destroy(cpl_plugin *);
00060 static int kmo_illumination(cpl_parameterlist *, cpl_frameset *);
00061 
00062 /*-----------------------------------------------------------------------------
00063  *                          Static variables
00064  *----------------------------------------------------------------------------*/
00065 
00066 static char kmo_illumination_description[] =
00067 "This recipe creates the spatial non-uniformity calibration frame needed for\n"
00068 "all three detectors. It must be called after the kmo_wave_cal-recipe, which\n"
00069 "generates the spectral calibration frame needed in this recipe. As input at\n"
00070 "least a sky, a master dark, a master flat and the spatial and spectral cali-\n"
00071 "bration frames are required.\n"
00072 "\n"
00073 "BASIC PARAMETERS:\n"
00074 "-----------------\n"
00075 "--imethod\n"
00076 "The interpolation method used for reconstruction.\n"
00077 "\n"
00078 "--range\n"
00079 "The spectral ranges to combine when collapsing the reconstructed cubes. e.g.\n"
00080 "\"x1_start,x1_end;x2_start,x2_end\" (microns)\n"
00081 "\n"
00082 "ADVANCED PARAMETERS\n"
00083 "-------------------\n"
00084 "--flux\n"
00085 "Specify if flux conservation should be applied.\n"
00086 "\n"
00087 "--add-all\n"
00088 "By default the first FLAT_SKY frame is omitted, since in the\n"
00089 "KMOS_spec_cal_skyflat template this is an acquisition frame to estimate the\n"
00090 "needed exposure time for the subsequent FLAT_SKY frames. If anyway all frames\n"
00091 "should be considered, set this parameter to TRUE.\n"
00092 "\n"
00093 "--neighborhoodRange\n"
00094 "Defines the range to search for neighbors during reconstruction\n"
00095 "\n"
00096 "--b_samples\n"
00097 "The number of samples in spectral direction for the reconstructed cube.\n"
00098 "Ideally this number should be greater than 2048, the detector size.\n"
00099 "\n"
00100 "--b_start\n"
00101 "--b_end\n"
00102 "Used to define manually the start and end wavelength for the reconstructed\n"
00103 "cube. By default the internally defined values are used.\n"
00104 "\n"
00105 "--cmethod\n"
00106 "Following methods of frame combination are available:\n"
00107 "   * 'ksigma' (Default)\n"
00108 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00109 "   are examined. If they deviate significantly, they will be rejected according\n"
00110 "   to the conditions:\n"
00111 "       val > mean + stdev * cpos_rej\n"
00112 "   and\n"
00113 "       val < mean - stdev * cneg_rej\n"
00114 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00115 "   parameters. In the first iteration median and percentile level are used.\n"
00116 "\n"
00117 "   * 'median'\n"
00118 "   At each pixel position the median is calculated.\n"
00119 "\n"
00120 "   * 'average'\n"
00121 "   At each pixel position the average is calculated.\n"
00122 "\n"
00123 "   * 'sum'\n"
00124 "   At each pixel position the sum is calculated.\n"
00125 "\n"
00126 "   * 'min_max'\n"
00127 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00128 "   --cmax and --cmin apply to this method.\n"
00129 "\n"
00130 "--cpos_rej\n"
00131 "--cneg_rej\n"
00132 "--citer\n"
00133 "see --cmethod='ksigma'\n"
00134 "\n"
00135 "--cmax\n"
00136 "--cmin\n"
00137 "see --cmethod='min_max'\n"
00138 "\n"
00139 "--pix_scale\n"
00140 "Change the pixel scale [arcsec]. Default of 0.2\" results into cubes of\n"
00141 "14x14pix, a scale of 0.1\" results into cubes of 28x28pix, etc.\n"
00142 "\n"
00143 "--suppress_extension\n"
00144 "If set to TRUE, the arbitrary filename extensions are supressed. If multiple\n"
00145 "products with the same category are produced, they will be numered consecutively\n"
00146 "starting from 0.\n"
00147 "\n"
00148 "-------------------------------------------------------------------------------\n"
00149 "  Input files:\n"
00150 "\n"
00151 "   DO                    KMOS                                                  \n"
00152 "   category              Type   Explanation                    Required #Frames\n"
00153 "   --------              -----  -----------                    -------- -------\n"
00154 "   FLAT_SKY               F2D   Sky exposures                     Y      1-n   \n"
00155 "                                (at least 3 frames recommended)                \n"
00156 "   MASTER_DARK            F2D   Master dark                       Y       1    \n"
00157 "   MASTER_FLAT            F2D   Master flat                       Y       1    \n"
00158 "   XCAL                   F2D   x calibration frame               Y       1    \n"
00159 "   YCAL                   F2D   y calibration frame               Y       1    \n"
00160 "   LCAL                   F2D   Wavelength calib. frame           Y       1    \n"
00161 "   WAVE_BAND              F2L   Table with start-/end-wavelengths Y       1    \n"
00162 "   FLAT_EDGE              F2L   Table with fitted slitlet edges   N      0,1   \n"
00163 "\n"
00164 "  Output files:\n"
00165 "\n"
00166 "   DO                    KMOS\n"
00167 "   category              Type   Explanation\n"
00168 "   --------              -----  -----------\n"
00169 "   ILLUM_CORR            F2I    Illumination calibration frame   \n"
00170 "   If FLAT_EDGE is provided: \n"
00171 "   SKYFLAT_EDGE          F2L    Frame containing parameters of fitted \n"
00172 "                                slitlets of all IFUs of all detectors\n"
00173 "-------------------------------------------------------------------------------\n"
00174 "\n";
00175 
00176 /*-----------------------------------------------------------------------------
00177  *                              Functions code
00178  *----------------------------------------------------------------------------*/
00179 
00196 int cpl_plugin_get_info(cpl_pluginlist *list)
00197 {
00198     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00199     cpl_plugin *plugin = &recipe->interface;
00200 
00201     cpl_plugin_init(plugin,
00202                         CPL_PLUGIN_API,
00203                         KMOS_BINARY_VERSION,
00204                         CPL_PLUGIN_TYPE_RECIPE,
00205                         "kmo_illumination",
00206                         "Create a calibration frame to correct spatial "
00207                         "non-uniformity of flatfield.",
00208                         kmo_illumination_description,
00209                         "Alex Agudo Berbel",
00210                         "kmos-spark@mpe.mpg.de",
00211                         kmos_get_license(),
00212                         kmo_illumination_create,
00213                         kmo_illumination_exec,
00214                         kmo_illumination_destroy);
00215 
00216     cpl_pluginlist_append(list, plugin);
00217 
00218     return 0;
00219 }
00220 
00228 static int kmo_illumination_create(cpl_plugin *plugin)
00229 {
00230     cpl_recipe *recipe;
00231     cpl_parameter *p;
00232 
00233     /* Check that the plugin is part of a valid recipe */
00234     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00235         recipe = (cpl_recipe *)plugin;
00236     else
00237         return -1;
00238 
00239     /* Create the parameters list in the cpl_recipe object */
00240     recipe->parameters = cpl_parameterlist_new();
00241 
00242     /* Fill the parameters list */
00243     /* --imethod */
00244     p = cpl_parameter_new_value("kmos.kmo_illumination.imethod",
00245                                 CPL_TYPE_STRING,
00246                                 "Method to use for interpolation: "
00247                                 "[\"NN\" (nearest neighbour), "
00248                                 "\"lwNN\" (linear weighted nearest neighbor),"
00249                                 "\"swNN\" (square weighted nearest neighbor)"
00250                                 "\"MS\" (Modified Shepard's method)"
00251                                 "\"CS\" (Cubic spline)]",
00252                                 "kmos.kmo_illumination",
00253                                 "CS");
00254     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "imethod");
00255     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00256     cpl_parameterlist_append(recipe->parameters, p);
00257 
00258     /* --neighborhoodRange */
00259     p = cpl_parameter_new_value("kmos.kmo_illumination.neighborhoodRange",
00260                                 CPL_TYPE_DOUBLE,
00261                                 "Defines the range to search for neighbors."
00262                                 "in pixels",
00263                                 "kmos.kmo_illumination",
00264                                 1.001);
00265     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "neighborhoodRange");
00266     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00267     cpl_parameterlist_append(recipe->parameters, p);
00268 
00269     /* --range */
00270     p = cpl_parameter_new_value("kmos.kmo_illumination.range",
00271                                 CPL_TYPE_STRING,
00272                                 "The spectral ranges to combine when collapsing"
00273                                 "the reconstructed cubes. e.g."
00274                                 "\"x1_start,x1_end;x2_start,x2_end\" (microns)",
00275                                 "kmos.kmo_illumination",
00276                                 "");
00277     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "range");
00278     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00279     cpl_parameterlist_append(recipe->parameters, p);
00280 
00281     /* --flux */
00282     p = cpl_parameter_new_value("kmos.kmo_illumination.flux",
00283                                 CPL_TYPE_BOOL,
00284                                 "TRUE: Apply flux conservation. FALSE: otherwise",
00285                                 "kmos.kmo_illumination",
00286                                 FALSE);
00287     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00288     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00289     cpl_parameterlist_append(recipe->parameters, p);
00290 
00291     /* --add-all */
00292     p = cpl_parameter_new_value("kmos.kmo_illumination.add-all",
00293                                 CPL_TYPE_BOOL,
00294                                 "FALSE: omit 1st FLAT_SKY frame (acquisition), "
00295                                 "TRUE: don't perform any checks, add them all",
00296                                 "kmos.kmo_illumination",
00297                                 FALSE);
00298     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "add-all");
00299     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00300     cpl_parameterlist_append(recipe->parameters, p);
00301 
00302     /* --pix_scale */
00303     p = cpl_parameter_new_value("kmos.kmo_illumination.pix_scale",
00304                                 CPL_TYPE_DOUBLE,
00305                                 "Change the pixel scale [arcsec]. "
00306                                 "Default of 0.2\" results into cubes of 14x14pix, "
00307                                 "a scale of 0.1\" results into cubes of 28x28pix, "
00308                                 "etc.",
00309                                 "kmos.kmo_illumination",
00310                                 KMOS_PIX_RESOLUTION);
00311     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pix_scale");
00312     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00313     cpl_parameterlist_append(recipe->parameters, p);
00314 
00315     /* --suppress_extension */
00316     p = cpl_parameter_new_value("kmos.kmo_illumination.suppress_extension",
00317                                 CPL_TYPE_BOOL,
00318                                 "Suppress arbitrary filename extension."
00319                                 "(TRUE (apply) or FALSE (don't apply)",
00320                                 "kmos.kmo_illumination",
00321                                 FALSE);
00322     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "suppress_extension");
00323     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00324     cpl_parameterlist_append(recipe->parameters, p);
00325 
00326     // add parameters for band-definition
00327     kmo_band_pars_create(recipe->parameters,
00328                          "kmos.kmo_illumination");
00329 
00330     // add parameters for combining
00331     return kmo_combine_pars_create(recipe->parameters,
00332                                    "kmos.kmo_illumination",
00333                                    DEF_REJ_METHOD,
00334                                    FALSE);
00335 }
00336 
00342 static int kmo_illumination_exec(cpl_plugin *plugin)
00343 {
00344     cpl_recipe  *recipe;
00345 
00346     /* Get the recipe out of the plugin */
00347     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00348         recipe = (cpl_recipe *)plugin;
00349     else return -1;
00350 
00351     return kmo_illumination(recipe->parameters, recipe->frames);
00352 }
00353 
00359 static int kmo_illumination_destroy(cpl_plugin *plugin)
00360 {
00361     cpl_recipe *recipe;
00362 
00363     /* Get the recipe out of the plugin */
00364     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00365         recipe = (cpl_recipe *)plugin;
00366     else return -1 ;
00367 
00368     cpl_parameterlist_delete(recipe->parameters);
00369     return 0 ;
00370 }
00371 
00386 static int kmo_illumination(cpl_parameterlist *parlist, cpl_frameset *frameset)
00387 {
00388     int              ret_val                    = 0,
00389                      nr_devices                 = 0,
00390                      ifu_nr                     = 0,
00391                      nx                         = 0,
00392                      ny                         = 0,
00393                      process_noise              = FALSE,
00394                      cmax                       = 0,
00395                      cmin                       = 0,
00396                      citer                      = 0,
00397                      *bounds                    = NULL,
00398                      cnt                        = 0,
00399                      qc_max_dev_id              = 0,
00400                      qc_max_nonunif_id          = 0,
00401                      flux                       = FALSE,
00402                      background                 = FALSE,
00403                      add_all_sky                = FALSE,
00404                      same_exptime               = TRUE,
00405                      suppress_extension         = FALSE,
00406                      has_flat_edge              = FALSE,
00407                      i = 0, j = 0, x = 0, y = 0, ix = 0, iy = 0, det_nr = 0, edgeNr = 0;
00408     const int        *punused_ifus              = NULL;
00409     float            *pbad_pix_mask             = NULL;
00410     double           exptime                    = 0.,
00411                      exptime1                   = 0.,
00412                      exptime2                   = 0.,
00413                      cpos_rej                   = 0.0,
00414                      cneg_rej                   = 0.0,
00415                      neighborhoodRange          = 1.001,
00416                      mean_data                  = 0.0,
00417                      ifu_crpix                  = 0.0,
00418                      ifu_crval                  = 0.0,
00419                      ifu_cdelt                  = 0.0,
00420                      qc_spat_unif               = 0.0,
00421                      qc_max_dev                 = 0.0,
00422                      qc_max_nonunif             = 0.0,
00423                      tmp_stdev                  = 0.0,
00424                      tmp_mean                   = 0.0,
00425                      rotangle                   = 0.0,
00426                      tmp_rotangle               = 0.0,
00427                      rotangle_found             = 0.0,
00428                      pix_scale                  = 0.0;
00429     char             *keyword                   = NULL,
00430                      *fn_lut                    = NULL,
00431                      *suffix                    = NULL,
00432                      *fn_suffix                 = NULL,
00433                      *extname                   = NULL,
00434                      *filter                    = NULL;
00435     const char       *method                    = NULL,
00436                      *cmethod                   = NULL,
00437                      *filter_id_l               = NULL,
00438                      *filter_id                 = NULL,
00439                      *ranges_txt                = NULL;
00440     cpl_array        *calTimestamp              = NULL,
00441                      **unused_ifus_before       = NULL,
00442                      **unused_ifus_after        = NULL;
00443     cpl_frame        *frame                     = NULL,
00444                      *xcalFrame                 = NULL,
00445                      *ycalFrame                 = NULL,
00446                      *lcalFrame                 = NULL;
00447     cpl_frameset     *frameset_sky              = NULL;
00448     cpl_image        *img_in                    = NULL,
00449                      *img_dark                  = NULL,
00450                      *img_dark_noise            = NULL,
00451                      *img_flat                  = NULL,
00452                      *img_flat_noise            = NULL,
00453                      *combined_data             = NULL,
00454                      *combined_noise            = NULL,
00455                      *xcal                      = NULL,
00456                      *ycal                      = NULL,
00457                      *lcal                      = NULL,
00458                      *bad_pix_mask              = NULL,
00459                      *data_ifu                  = NULL,
00460                      *noise_ifu                 = NULL,
00461                      **stored_data_images       = NULL,
00462                      **stored_noise_images      = NULL;
00463     cpl_imagelist    *cube_data                 = NULL,
00464                      *cube_noise                = NULL,
00465                      *detector_in               = NULL,
00466                      **stored_data_cubes        = NULL,
00467                      **stored_noise_cubes       = NULL;
00468     cpl_matrix       **edgepars                 = NULL;
00469     cpl_propertylist *main_header               = NULL,
00470                      *tmp_header                = NULL,
00471                      *sub_header                = NULL,
00472                      **stored_sub_data_headers  = NULL,
00473                      **stored_sub_noise_headers = NULL;
00474     cpl_table        *band_table                = NULL,
00475                      ***edge_table_sky          = NULL,
00476                      **edge_table_flat          = NULL;
00477     cpl_vector       *ranges                    = NULL,
00478                      *identified_slices         = NULL,
00479                      *calAngles                 = NULL,
00480                      **slitlet_ids              = NULL,
00481                      *shift_vec                 = NULL,
00482                      *edge_vec                  = NULL;
00483     main_fits_desc   desc_sky,
00484                      desc_dark,
00485                      desc_flat,
00486                      desc_xcal,
00487                      desc_ycal,
00488                      desc_lcal;
00489     gridDefinition   gd;
00490 
00491     KMO_TRY
00492     {
00493         kmo_init_fits_desc(&desc_sky);
00494         kmo_init_fits_desc(&desc_dark);
00495         kmo_init_fits_desc(&desc_flat);
00496         kmo_init_fits_desc(&desc_xcal);
00497         kmo_init_fits_desc(&desc_ycal);
00498         kmo_init_fits_desc(&desc_lcal);
00499 
00500         /* --- check input --- */
00501         KMO_TRY_ASSURE((parlist != NULL) &&
00502                        (frameset != NULL),
00503                        CPL_ERROR_NULL_INPUT,
00504                        "Not all input data is provided!");
00505 
00506         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, FLAT_SKY) >= 1,
00507                        CPL_ERROR_ILLEGAL_INPUT,
00508                        "One or more FLAT_SKY frames are required!");
00509 
00510         if (cpl_frameset_count_tags(frameset, FLAT_SKY) < 3) {
00511             cpl_msg_warning(cpl_func, "It is recommended to provide at least "
00512                                       "3 FLAT_SKY frames!");
00513         }
00514 
00515         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_DARK) == 1,
00516                        CPL_ERROR_ILLEGAL_INPUT,
00517                        "Exactly one MASTER_DARK frame is required!");
00518 
00519         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_FLAT) == 1,
00520                        CPL_ERROR_ILLEGAL_INPUT,
00521                        "Exactly one MASTER_FLAT frame is required!");
00522 
00523         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, XCAL) == 1,
00524                        CPL_ERROR_ILLEGAL_INPUT,
00525                        "Exactly one XCAL frame is required!");
00526 
00527         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, YCAL) == 1,
00528                        CPL_ERROR_ILLEGAL_INPUT,
00529                        "Exactly one YCAL frame is required!");
00530 
00531         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, LCAL) == 1,
00532                        CPL_ERROR_ILLEGAL_INPUT,
00533                        "Exactly one LCAL frame is required!");
00534 
00535         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, WAVE_BAND) == 1,
00536                        CPL_ERROR_ILLEGAL_INPUT,
00537                        "Exactly one WAVE_BAND frame is required!");
00538 
00539         KMO_TRY_ASSURE((cpl_frameset_count_tags(frameset, FLAT_EDGE) == 1) ||
00540                        (cpl_frameset_count_tags(frameset, FLAT_EDGE) == 0),
00541                        CPL_ERROR_ILLEGAL_INPUT,
00542                        "Exactly one FLAT_EDGE frame is required!");
00543 
00544         has_flat_edge = cpl_frameset_count_tags(frameset, FLAT_EDGE);
00545 
00546         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_illumination") == 1,
00547                        CPL_ERROR_ILLEGAL_INPUT,
00548                        "Cannot identify RAW and CALIB frames!");
00549 
00550         /* --- get parameters --- */
00551         cpl_msg_info("", "--- Parameter setup for kmo_illumination ---");
00552 
00553         KMO_TRY_EXIT_IF_NULL(
00554             method = kmo_dfs_get_parameter_string(parlist,
00555                                               "kmos.kmo_illumination.imethod"));
00556 
00557         KMO_TRY_ASSURE((strcmp(method, "NN") == 0) ||
00558                        (strcmp(method, "lwNN") == 0) ||
00559                        (strcmp(method, "swNN") == 0) ||
00560                        (strcmp(method, "MS") == 0) ||
00561                        (strcmp(method, "CS") == 0),
00562                        CPL_ERROR_ILLEGAL_INPUT,
00563                        "method must be either \"NN\", \"lwNN\", "
00564                        "\"swNN\", \"MS\" or \"CS\"!");
00565 
00566         KMO_TRY_EXIT_IF_ERROR(
00567             kmo_dfs_print_parameter_help(parlist,
00568                                         "kmos.kmo_illumination.imethod"));
00569 
00570         neighborhoodRange = kmo_dfs_get_parameter_double(parlist,
00571                 "kmos.kmo_illumination.neighborhoodRange");
00572         KMO_TRY_CHECK_ERROR_STATE();
00573 
00574         KMO_TRY_ASSURE(neighborhoodRange > 0.0,
00575                 CPL_ERROR_ILLEGAL_INPUT,
00576                 "neighborhoodRange must be greater than 0.0");
00577 
00578         KMO_TRY_EXIT_IF_ERROR(
00579             kmo_dfs_print_parameter_help(parlist,
00580                                     "kmos.kmo_illumination.neighborhoodRange"));
00581 
00582         ranges_txt = kmo_dfs_get_parameter_string(parlist,
00583                                                   "kmos.kmo_illumination.range");
00584         KMO_TRY_CHECK_ERROR_STATE();
00585 
00586         KMO_TRY_EXIT_IF_ERROR(
00587             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.range"));
00588 
00589         ranges = kmo_identify_ranges(ranges_txt);
00590         KMO_TRY_CHECK_ERROR_STATE();
00591 
00592         flux = kmo_dfs_get_parameter_bool(parlist,
00593                                           "kmos.kmo_illumination.flux");
00594 
00595         KMO_TRY_ASSURE((flux == 0) ||
00596                        (flux == 1),
00597                        CPL_ERROR_ILLEGAL_INPUT,
00598                        "flux must be either FALSE or TRUE!");
00599 
00600         KMO_TRY_EXIT_IF_ERROR(
00601             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.flux"));
00602 
00603         add_all_sky = kmo_dfs_get_parameter_bool(parlist,
00604                                                  "kmos.kmo_illumination.add-all");
00605 
00606         KMO_TRY_ASSURE((add_all_sky == 0) ||
00607                        (add_all_sky == 1),
00608                        CPL_ERROR_ILLEGAL_INPUT,
00609                        "add_all must be either FALSE or TRUE!");
00610 
00611         KMO_TRY_EXIT_IF_ERROR(
00612             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.add-all"));
00613 
00614         pix_scale = kmo_dfs_get_parameter_double(parlist,
00615                                         "kmos.kmo_illumination.pix_scale");
00616         KMO_TRY_CHECK_ERROR_STATE();
00617         KMO_TRY_EXIT_IF_ERROR(
00618            kmo_dfs_print_parameter_help(parlist,
00619                                        "kmos.kmo_illumination.pix_scale"));
00620         KMO_TRY_ASSURE((pix_scale >= 0.01) &&
00621                        (pix_scale <= 0.4),
00622                        CPL_ERROR_ILLEGAL_INPUT,
00623                        "pix_scale must be between 0.01 and 0.4 (results in cubes "
00624                        "with 7x7 to 280x280 pixels)!");
00625 
00626         suppress_extension = kmo_dfs_get_parameter_bool(parlist,
00627                                           "kmos.kmo_illumination.suppress_extension");
00628         KMO_TRY_CHECK_ERROR_STATE();
00629         KMO_TRY_EXIT_IF_ERROR(
00630             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.suppress_extension"));
00631 
00632         KMO_TRY_ASSURE((suppress_extension == TRUE) || (suppress_extension == FALSE),
00633                        CPL_ERROR_ILLEGAL_INPUT,
00634                        "suppress_extension must be TRUE or FALSE!");
00635 
00636         kmo_band_pars_load(parlist, "kmos.kmo_illumination");
00637 
00638         KMO_TRY_EXIT_IF_ERROR(
00639             kmo_combine_pars_load(parlist,
00640                                   "kmos.kmo_illumination",
00641                                   &cmethod,
00642                                   &cpos_rej,
00643                                   &cneg_rej,
00644                                   &citer,
00645                                   &cmin,
00646                                   &cmax,
00647                                   FALSE));
00648         cpl_msg_info("", "-------------------------------------------");
00649 
00650         // check if filter_id, grating_id and rotator offset match for all
00651         // detectors
00652         KMO_TRY_EXIT_IF_ERROR(
00653             kmo_check_frameset_setup(frameset, FLAT_SKY,
00654                                        TRUE, FALSE, TRUE));
00655         KMO_TRY_EXIT_IF_ERROR(
00656             kmo_check_frame_setup(frameset, FLAT_SKY, XCAL,
00657                                        TRUE, FALSE, TRUE));
00658         KMO_TRY_EXIT_IF_ERROR(
00659             kmo_check_frame_setup(frameset, XCAL, YCAL,
00660                                        TRUE, FALSE, TRUE));
00661         KMO_TRY_EXIT_IF_ERROR(
00662             kmo_check_frame_setup(frameset, XCAL, LCAL,
00663                                        TRUE, FALSE, TRUE));
00664         KMO_TRY_EXIT_IF_ERROR(
00665             kmo_check_frame_setup(frameset, XCAL, MASTER_FLAT,
00666                                        TRUE, FALSE, TRUE));
00667 
00668         KMO_TRY_EXIT_IF_NULL(
00669             frame = kmo_dfs_get_frame(frameset, XCAL));
00670         KMO_TRY_EXIT_IF_NULL(
00671             suffix = kmo_dfs_get_suffix(frame, TRUE, FALSE));
00672 
00673         KMO_TRY_EXIT_IF_ERROR(
00674             kmo_check_frame_setup_md5_xycal(frameset));
00675         KMO_TRY_EXIT_IF_ERROR(
00676             kmo_check_frame_setup_md5(frameset));
00677 
00678         cpl_msg_info("", "Detected instrument setup:   %s", suffix+1);
00679         cpl_msg_info("", "(grating 1, 2 & 3)");
00680 
00681         // check which IFUs are active for all frames
00682         KMO_TRY_EXIT_IF_NULL(
00683             unused_ifus_before = kmo_get_unused_ifus(frameset, 0, 0));
00684 
00685         KMO_TRY_EXIT_IF_NULL(
00686             unused_ifus_after = kmo_duplicate_unused_ifus(unused_ifus_before));
00687 
00688         kmo_print_unused_ifus(unused_ifus_before, FALSE);
00689 
00690         // load desc for MASTER_DARK and check
00691         KMO_TRY_EXIT_IF_NULL(
00692             frame = kmo_dfs_get_frame(frameset, MASTER_DARK));
00693         desc_dark = kmo_identify_fits_header(
00694                     cpl_frame_get_filename(frame));
00695         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_DARK frame doesn't seem to "
00696                                       "be in KMOS-format!");
00697         KMO_TRY_ASSURE((desc_dark.nr_ext == 2*KMOS_NR_DETECTORS) &&
00698                        (desc_dark.ex_badpix == FALSE) &&
00699                        (desc_dark.fits_type == f2d_fits) &&
00700                        (desc_dark.frame_type == detector_frame),
00701                        CPL_ERROR_ILLEGAL_INPUT,
00702                        "MASTER_DARK isn't in the correct format!!!");
00703         nx = desc_dark.naxis1;
00704         ny = desc_dark.naxis2;
00705 
00706         // load desc for MASTER_FLAT and check
00707         KMO_TRY_EXIT_IF_NULL(
00708             frame = kmo_dfs_get_frame(frameset, MASTER_FLAT));
00709         desc_flat = kmo_identify_fits_header(cpl_frame_get_filename(frame));
00710         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_FLAT frame doesn't seem to "
00711                                       "be in KMOS-format!");
00712         KMO_TRY_ASSURE((desc_flat.nr_ext % (2*KMOS_NR_DETECTORS) == 0) &&
00713                        (desc_flat.ex_badpix == FALSE) &&
00714                        (desc_flat.fits_type == f2d_fits) &&
00715                        (desc_flat.frame_type == detector_frame),
00716                        CPL_ERROR_ILLEGAL_INPUT,
00717                        "MASTER_FLAT isn't in the correct format!!!");
00718 
00719         // load desc for XCAL and check
00720         KMO_TRY_EXIT_IF_NULL(
00721             xcalFrame = kmo_dfs_get_frame(frameset, XCAL));
00722         desc_xcal = kmo_identify_fits_header(cpl_frame_get_filename(xcalFrame));
00723         KMO_TRY_CHECK_ERROR_STATE_MSG("XCAL frame doesn't seem to "
00724                                       "be in KMOS-format!");
00725         KMO_TRY_ASSURE((desc_xcal.nr_ext % KMOS_NR_DETECTORS == 0) &&
00726                        (desc_xcal.ex_badpix == FALSE) &&
00727                        (desc_xcal.fits_type == f2d_fits) &&
00728                        (desc_xcal.frame_type == detector_frame),
00729                        CPL_ERROR_ILLEGAL_INPUT,
00730                        "XCAL isn't in the correct format!!!");
00731         KMO_TRY_ASSURE((desc_xcal.naxis1 == nx) &&
00732                        (desc_xcal.naxis2 == ny),
00733                        CPL_ERROR_ILLEGAL_INPUT,
00734                        "MASTER_DARK and XCAL frame haven't same dimensions! "
00735                        "(x,y): (%d,%d) vs (%d,%d)",
00736                        nx, ny, desc_xcal.naxis1, desc_xcal.naxis2);
00737 
00738         nr_devices = desc_xcal.nr_ext;
00739 
00740         // load desc for YCAL and check
00741         KMO_TRY_EXIT_IF_NULL(
00742             ycalFrame = kmo_dfs_get_frame(frameset, YCAL));
00743         desc_ycal = kmo_identify_fits_header(cpl_frame_get_filename(ycalFrame));
00744         KMO_TRY_CHECK_ERROR_STATE_MSG("YCAL frame doesn't seem to "
00745                                       "be in KMOS-format!");
00746         KMO_TRY_ASSURE((desc_ycal.nr_ext == desc_xcal.nr_ext) &&
00747                        (desc_ycal.ex_badpix == desc_xcal.ex_badpix) &&
00748                        (desc_ycal.fits_type == desc_xcal.fits_type) &&
00749                        (desc_ycal.frame_type == desc_xcal.frame_type),
00750                        CPL_ERROR_ILLEGAL_INPUT,
00751                        "YCAL isn't in the correct format!!!");
00752         KMO_TRY_ASSURE((desc_ycal.naxis1 == desc_xcal.naxis1) &&
00753                        (desc_ycal.naxis2 == desc_xcal.naxis2),
00754                        CPL_ERROR_ILLEGAL_INPUT,
00755                        "MASTER_DARK and YCAL frame haven't same dimensions! "
00756                        "(x,y): (%d,%d) vs (%d,%d)",
00757                        nx, ny, desc_ycal.naxis1, desc_ycal.naxis2);
00758 
00759         // load desc for LCAL and check
00760         KMO_TRY_EXIT_IF_NULL(
00761             lcalFrame = kmo_dfs_get_frame(frameset, LCAL));
00762         desc_lcal = kmo_identify_fits_header(cpl_frame_get_filename(lcalFrame));
00763         KMO_TRY_CHECK_ERROR_STATE_MSG("LCAL frame doesn't seem to "
00764                                       "be in KMOS-format!");
00765         KMO_TRY_ASSURE((desc_lcal.ex_badpix == desc_xcal.ex_badpix) &&
00766                        (desc_lcal.fits_type == desc_xcal.fits_type) &&
00767                        (desc_lcal.frame_type == desc_xcal.frame_type),
00768                        CPL_ERROR_ILLEGAL_INPUT,
00769                        "LCAL isn't in the correct format!!!");
00770         KMO_TRY_ASSURE((desc_lcal.naxis1 == desc_xcal.naxis1) &&
00771                        (desc_lcal.naxis2 == desc_xcal.naxis2),
00772                        CPL_ERROR_ILLEGAL_INPUT,
00773                        "MASTER_DARK and LCAL frame haven't same dimensions! "
00774                        "(x,y): (%d,%d) vs (%d,%d)",
00775                        nx, ny, desc_lcal.naxis1, desc_lcal.naxis2);
00776         KMO_TRY_EXIT_IF_NULL(
00777             tmp_header = kmo_dfs_load_primary_header(frameset, LCAL));
00778 
00779         // load desc for FLAT_SKY and check
00780         nr_devices = KMOS_NR_DETECTORS;
00781         KMO_TRY_EXIT_IF_NULL(
00782             frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00783 
00784         KMO_TRY_EXIT_IF_NULL(
00785             main_header = kmclipm_propertylist_load(
00786                                          cpl_frame_get_filename(frame), 0));
00787         rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00788         KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00789         kmclipm_strip_angle(&rotangle);
00790         cpl_propertylist_delete(main_header); main_header = NULL;
00791 
00792         cnt = 1;
00793         while (frame != NULL) {
00794             KMO_TRY_EXIT_IF_NULL(
00795                 main_header = kmclipm_propertylist_load(
00796                                              cpl_frame_get_filename(frame), 0));
00797 
00798             desc_sky = kmo_identify_fits_header(
00799                         cpl_frame_get_filename(frame));
00800             KMO_TRY_CHECK_ERROR_STATE_MSG("FLAT_SKY frame doesn't seem to "
00801                                           "be in KMOS-format!");
00802             KMO_TRY_ASSURE((desc_sky.nr_ext == 3) &&
00803                            (desc_sky.ex_badpix == FALSE) &&
00804                            (desc_sky.fits_type == raw_fits) &&
00805                            (desc_sky.frame_type == detector_frame),
00806                            CPL_ERROR_ILLEGAL_INPUT,
00807                            "FLAT_SKY isn't in the correct format!!!");
00808             KMO_TRY_ASSURE((desc_sky.naxis1 == nx) &&
00809                            (desc_sky.naxis2 == ny) &&
00810                            (desc_sky.nr_ext == nr_devices),
00811                            CPL_ERROR_ILLEGAL_INPUT,
00812                            "MASTER_DARK and FLAT_SKY (no. %d) frame haven't "
00813                            "same dimensions! (x,y): (%d,%d) vs (%d,%d)",
00814                            cnt, nx, ny, desc_flat.naxis1, desc_flat.naxis2);
00815             kmo_free_fits_desc(&desc_sky);
00816             kmo_init_fits_desc(&desc_sky);
00817 
00818             KMO_TRY_ASSURE(
00819                 (kmo_check_lamp(main_header, INS_LAMP1_ST) == FALSE) &&
00820                 (kmo_check_lamp(main_header, INS_LAMP2_ST) == FALSE) &&
00821                 (kmo_check_lamp(main_header, INS_LAMP3_ST) == FALSE) &&
00822                 (kmo_check_lamp(main_header, INS_LAMP4_ST) == FALSE),
00823                 CPL_ERROR_ILLEGAL_INPUT,
00824                 "All lamps must be switched off for the FLAT_SKY frames!");
00825 
00826             // assert that filters have correct IDs and that all detectors of
00827             // all input frames have the same filter set
00828             for (i = 1; i <= KMOS_NR_DETECTORS; i++) {
00829                 // ESO INS FILTi ID
00830                 KMO_TRY_EXIT_IF_NULL(
00831                     keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, i, IFU_FILTID_POSTFIX));
00832                 KMO_TRY_EXIT_IF_NULL(
00833                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00834 
00835                 KMO_TRY_EXIT_IF_NULL(
00836                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00837                 cpl_free(keyword); keyword = NULL;
00838 
00839                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00840                                (strcmp(filter_id, "YJ") == 0) ||
00841                                (strcmp(filter_id, "H") == 0) ||
00842                                (strcmp(filter_id, "K") == 0) ||
00843                                (strcmp(filter_id, "HK") == 0),
00844                                CPL_ERROR_ILLEGAL_INPUT,
00845                                "Filter ID in primary header must be either 'IZ', "
00846                                "'YJ', 'H', 'K' or " "'HK' !");
00847 
00848                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00849                                CPL_ERROR_ILLEGAL_INPUT,
00850                                "Filter IDs must be the same for FLAT_SKY frame"
00851                                " and lcal frame!"
00852                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00853                                i, cpl_frame_get_filename(frame),
00854                                filter_id, filter_id_l);
00855 
00856                 // ESO INS GRATi ID
00857                 KMO_TRY_EXIT_IF_NULL(
00858                     keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, i, IFU_GRATID_POSTFIX));
00859                 KMO_TRY_EXIT_IF_NULL(
00860                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00861 
00862                 KMO_TRY_EXIT_IF_NULL(
00863                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00864                 cpl_free(keyword); keyword = NULL;
00865 
00866                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00867                                (strcmp(filter_id, "YJ") == 0) ||
00868                                (strcmp(filter_id, "H") == 0) ||
00869                                (strcmp(filter_id, "K") == 0) ||
00870                                (strcmp(filter_id, "HK") == 0),
00871                                CPL_ERROR_ILLEGAL_INPUT,
00872                                "Grating ID in primary header must be either "
00873                                "'IZ', 'YJ', 'H', 'K' or " "'HK' !");
00874 
00875                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00876                                CPL_ERROR_ILLEGAL_INPUT,
00877                                "Grating IDs must be the same for FLAT_SKY frame"
00878                                " and lcal frame!"
00879                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00880                                i, cpl_frame_get_filename(frame),
00881                                filter_id, filter_id_l);
00882 
00883                 tmp_rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00884                 KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00885                 kmclipm_strip_angle(&tmp_rotangle);
00886                 KMO_TRY_ASSURE((abs(rotangle - tmp_rotangle) < 10.0) ||
00887                                (abs(rotangle - tmp_rotangle) > 360.-10.) ,
00888                         CPL_ERROR_ILLEGAL_INPUT,
00889                         "OCS ROT NAANGLE of sky flat frames differ to much: %f %f",
00890                         rotangle, tmp_rotangle);
00891             }
00892             cpl_propertylist_delete(main_header); main_header = NULL;
00893 
00894             // get next FLAT_SKY frame
00895             frame = kmo_dfs_get_frame(frameset, NULL);
00896             KMO_TRY_CHECK_ERROR_STATE();
00897             cnt++;
00898         }
00899 
00900         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00901 
00902         //
00903         // noise will be propagated when:
00904         // MASTER_DARK and MASTER_FLAT have noise extensions and if at least
00905         // 2 FLAT_SKY frames are provided.
00906         // Otherwise noise will be ignored.
00907         //
00908         if (desc_dark.ex_noise &&
00909             desc_flat.ex_noise &&
00910             (cpl_frameset_count_tags(frameset, FLAT_SKY) >= 2)) {
00911             process_noise = TRUE;
00912         }
00913 
00914         if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00915             cpl_msg_warning(cpl_func, "cmethod is changed to 'average' "
00916                             "since there is only one input frame! (The output "
00917                             "file won't have any noise extensions)");
00918 
00919             cmethod = "average";
00920         }
00921 
00922         //
00923         // Check whether 1st FLAT_SKY should be omitted
00924         //
00925         KMO_TRY_EXIT_IF_NULL(
00926             frameset_sky = cpl_frameset_new());
00927 
00928         if (add_all_sky) {
00929             // just add all FLAT_SKY frames without check
00930             frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00931             while (frame != NULL) {
00932                 KMO_TRY_EXIT_IF_ERROR(
00933                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00934                 frame = kmo_dfs_get_frame(frameset, NULL);
00935             }
00936             cpl_msg_info("", "Add all FLAT_SKY without checking for acquisition frame.");
00937         } else {
00938             // check if 1st FLAT_SKY has different exposure time and whether to omit it
00939             KMO_TRY_EXIT_IF_NULL(
00940                 frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00941 
00942             if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00943                 // just one FLAT_SKY, always add
00944                 KMO_TRY_EXIT_IF_ERROR(
00945                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00946                 KMO_TRY_CHECK_ERROR_STATE();
00947             } else {
00948                 // several FLAT_SKY frames, check exptime
00949 
00950                 // get exptime 1
00951                 KMO_TRY_EXIT_IF_NULL(
00952                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00953                 exptime1 = cpl_propertylist_get_double(main_header, EXPTIME);
00954                 KMO_TRY_CHECK_ERROR_STATE();
00955                 cpl_propertylist_delete(main_header); main_header = NULL;
00956 
00957                 // get exptime 2
00958                 frame = kmo_dfs_get_frame(frameset, NULL);
00959                 KMO_TRY_EXIT_IF_NULL(
00960                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00961                 exptime2 = cpl_propertylist_get_double(main_header, EXPTIME);
00962                 KMO_TRY_CHECK_ERROR_STATE();
00963                 cpl_propertylist_delete(main_header); main_header = NULL;
00964 
00965                 // loop remaining frames
00966                 same_exptime = TRUE;
00967                 frame = kmo_dfs_get_frame(frameset, NULL);
00968                 while (same_exptime && (frame != NULL)) {
00969                     KMO_TRY_EXIT_IF_NULL(
00970                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00971                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00972                     KMO_TRY_CHECK_ERROR_STATE();
00973                     cpl_propertylist_delete(main_header); main_header = NULL;
00974                     if (fabs(exptime-exptime2) > 0.01) {
00975                         // not same
00976                         same_exptime = FALSE;
00977                     }
00978                     frame = kmo_dfs_get_frame(frameset, NULL);
00979                 }
00980 
00981                 if (same_exptime) {
00982                     // frame [2,n] have same exptime, add them
00983                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00984                     KMO_TRY_EXIT_IF_NULL(
00985                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00986                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00987                     KMO_TRY_CHECK_ERROR_STATE();
00988                     cpl_propertylist_delete(main_header); main_header = NULL;
00989                     cpl_msg_info("", "Omit FLAT_SKY: %s with EXPTIME of %g sec (acquisition), other frame(s) have EXPTIME of %g sec", cpl_frame_get_filename(frame), exptime, exptime2);
00990                     frame = kmo_dfs_get_frame(frameset, NULL);
00991                     while (frame != NULL) {
00992                         KMO_TRY_EXIT_IF_ERROR(
00993                             cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00994                         frame = kmo_dfs_get_frame(frameset, NULL);
00995                     }
00996                     if (fabs(exptime1-exptime2) < 0.01) {
00997                         cpl_msg_warning("", "The 1st FLAT_SKY has the same exposure time as the following ones. "
00998                                             "It has anyway been omitted since we assume it is an acquisition frame. "
00999                                             "If you want to add it anyway call this recipe with the --add-all parameter");
01000                     }
01001                 } else {
01002                     cpl_msg_error("", "The exposure times of the FLAT_SKY frames don't match!");
01003                     cpl_msg_error("", "We assume that the 1st frame is an acquisition frame and would be omitted.");
01004                     cpl_msg_error("", "The following frames should have the same exposure time if they originate from the same template.");
01005                     cpl_msg_error("", "If you want to reduce them anyway call this recipe with the --add-all parameter");
01006                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
01007                     while (frame != NULL) {
01008                         KMO_TRY_EXIT_IF_NULL(
01009                             main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
01010                         exptime = cpl_propertylist_get_double(main_header, EXPTIME);
01011                         KMO_TRY_CHECK_ERROR_STATE();
01012                         cpl_propertylist_delete(main_header); main_header = NULL;
01013                         cpl_msg_error("", "FLAT_SKY: %s, EXPTIME: %g", cpl_frame_get_filename(frame), exptime);
01014                         frame = kmo_dfs_get_frame(frameset, NULL);
01015                     }
01016                     cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_INPUT);
01017                     KMO_TRY_CHECK_ERROR_STATE();
01018                 }
01019             }
01020         }
01021 
01022         KMO_TRY_EXIT_IF_NULL(
01023             frame = kmo_dfs_get_frame(frameset_sky, FLAT_SKY));
01024         KMO_TRY_EXIT_IF_NULL(
01025             main_header = kmo_dfs_load_primary_header(frameset_sky, FLAT_SKY));
01026         KMO_TRY_EXIT_IF_NULL(
01027             keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, 1, IFU_GRATID_POSTFIX));
01028         KMO_TRY_EXIT_IF_NULL(
01029             filter = cpl_sprintf("%s", cpl_propertylist_get_string(main_header, keyword)));
01030         cpl_free(keyword); keyword = NULL;
01031 
01032         //
01033         // set default band-specific ranges for collapsing
01034         //
01035         if (ranges == NULL) {
01036             if (strcmp(filter, "IZ") == 0) {
01037                 ranges_txt = "0.81,1.05";
01038             } else if (strcmp(filter, "YJ") == 0) {
01039                 ranges_txt = "1.025,1.3";
01040             } else if (strcmp(filter, "H") == 0) {
01041                 ranges_txt = "1.5,1.7";
01042             } else if (strcmp(filter, "K") == 0) {
01043                 ranges_txt = "2.1,2.35";
01044             } else if (strcmp(filter, "HK") == 0) {
01045                 ranges_txt = "1.5,1.7;2.1,2.35";
01046 //                ranges_txt = "1.5,1.7";
01047             } else {
01048                 KMO_TRY_ASSURE(1 == 0,
01049                                CPL_ERROR_ILLEGAL_INPUT,
01050                                "We really shouldn't get here...");
01051             }
01052             cpl_msg_info("", "Spectral range to collapse has been set to %s um for this band.", ranges_txt);
01053             ranges = kmo_identify_ranges(ranges_txt);
01054             KMO_TRY_CHECK_ERROR_STATE();
01055         }
01056 
01057         // setup grid definition, wavelength start and end points will be set
01058         // in the detector loop
01059         KMO_TRY_EXIT_IF_ERROR(
01060             kmclipm_setup_grid(&gd, method, neighborhoodRange, pix_scale));
01061 
01062         // create filename for LUT
01063         KMO_TRY_EXIT_IF_NULL(
01064             fn_lut = cpl_sprintf("%s%s", "lut", suffix));
01065 
01066         // extract bounds
01067         KMO_TRY_EXIT_IF_NULL(
01068             tmp_header = kmclipm_propertylist_load(
01069                                          cpl_frame_get_filename(xcalFrame), 0));
01070         KMO_TRY_EXIT_IF_NULL(
01071             bounds = kmclipm_extract_bounds(tmp_header));
01072         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
01073 
01074         // get timestamps of xcal, ycal & lcal
01075         KMO_TRY_EXIT_IF_NULL(
01076             calTimestamp = kmo_get_timestamps(xcalFrame, ycalFrame, lcalFrame));
01077 
01078         // create arrays to hold reconstructed data and noise cubes and
01079         // their headers
01080         KMO_TRY_EXIT_IF_NULL(
01081             stored_data_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01082                                                             sizeof(cpl_imagelist*)));
01083         KMO_TRY_EXIT_IF_NULL(
01084             stored_noise_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01085                                                              sizeof(cpl_imagelist*)));
01086         KMO_TRY_EXIT_IF_NULL(
01087             stored_data_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01088                                                          sizeof(cpl_image*)));
01089         KMO_TRY_EXIT_IF_NULL(
01090             stored_noise_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01091                                                           sizeof(cpl_image*)));
01092         KMO_TRY_EXIT_IF_NULL(
01093             stored_sub_data_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01094                                                                      sizeof(cpl_propertylist*)));
01095         KMO_TRY_EXIT_IF_NULL(
01096             stored_sub_noise_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01097                                                                       sizeof(cpl_propertylist*)));
01098         KMO_TRY_EXIT_IF_NULL(
01099             edge_table_sky = (cpl_table***)cpl_calloc(KMOS_NR_DETECTORS,
01100                                                       sizeof(cpl_table**)));
01101         KMO_TRY_EXIT_IF_NULL(
01102             edge_table_flat = (cpl_table**)cpl_calloc(KMOS_IFUS_PER_DETECTOR,
01103                                                       sizeof(cpl_table*)));
01104         KMO_TRY_EXIT_IF_NULL(
01105             calAngles = cpl_vector_new(3));
01106 
01107         //
01108         // loop through all detectors
01109         //
01110         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01111             cpl_msg_info("","Processing detector No. %d", det_nr);
01112 
01113             KMO_TRY_EXIT_IF_NULL(
01114                 detector_in = cpl_imagelist_new());
01115 
01116             // load data of det_nr of all FLAT_SKY frames into an imagelist
01117             KMO_TRY_EXIT_IF_NULL(
01118                 img_in = kmo_dfs_load_image(frameset_sky, FLAT_SKY, det_nr, FALSE, TRUE, NULL));
01119 
01120             cnt = 0;
01121             while (img_in != NULL) {
01122                 cpl_imagelist_set(detector_in, img_in, cnt);
01123                 KMO_TRY_CHECK_ERROR_STATE();
01124 
01125                 /* load same extension of next FLAT_SKY frame*/
01126                 img_in = kmo_dfs_load_image(frameset_sky, NULL, det_nr, FALSE, TRUE, NULL);
01127                 KMO_TRY_CHECK_ERROR_STATE();
01128 
01129                 cnt++;
01130             }
01131 
01132             //
01133             // process imagelist
01134             //
01135 
01136             // combine imagelist (data only) and create noise (stdev of data)
01137             cpl_msg_info("","Combining frames...");
01138             if (process_noise) {
01139                 KMO_TRY_EXIT_IF_ERROR(
01140                     kmclipm_combine_frames(detector_in,
01141                                            NULL,
01142                                            NULL,
01143                                            cmethod,
01144                                            cpos_rej,
01145                                            cneg_rej,
01146                                            citer,
01147                                            cmax,
01148                                            cmin,
01149                                            &combined_data,
01150                                            &combined_noise,
01151                                            -1.0));
01152             } else {
01153                 KMO_TRY_EXIT_IF_ERROR(
01154                     kmclipm_combine_frames(detector_in,
01155                                            NULL,
01156                                            NULL,
01157                                            cmethod,
01158                                            cpos_rej,
01159                                            cneg_rej,
01160                                            citer,
01161                                            cmax,
01162                                            cmin,
01163                                            &combined_data,
01164                                            NULL,
01165                                            -1.0));
01166             }
01167 
01168             if (kmclipm_omit_warning_one_slice > 10) {
01169                 cpl_msg_warning(cpl_func, "Previous warning (number of "
01170                                           "identified slices) occured %d times.",
01171                                 kmclipm_omit_warning_one_slice);
01172                 kmclipm_omit_warning_one_slice = FALSE;
01173             }
01174 
01175             cpl_imagelist_delete(detector_in); detector_in = NULL;
01176 
01177             // load calibration files
01178             KMO_TRY_EXIT_IF_NULL(
01179                 xcal = kmo_dfs_load_cal_image(frameset, XCAL, det_nr, FALSE, rotangle,
01180                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01181 
01182             KMO_TRY_EXIT_IF_ERROR(
01183                 cpl_vector_set(calAngles, 0, rotangle_found));
01184             KMO_TRY_EXIT_IF_NULL(
01185                 ycal = kmo_dfs_load_cal_image(frameset, YCAL, det_nr, FALSE, rotangle,
01186                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01187             KMO_TRY_EXIT_IF_ERROR(
01188                 cpl_vector_set(calAngles, 1, rotangle_found));
01189             KMO_TRY_EXIT_IF_NULL(
01190                 lcal = kmo_dfs_load_cal_image(frameset, LCAL, det_nr, FALSE, rotangle,
01191                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01192             KMO_TRY_EXIT_IF_ERROR(
01193                 cpl_vector_set(calAngles, 2, rotangle_found));
01194 
01195             // load bad pixel mask from XCAL and set NaNs to 0 and all other values to 1
01196             KMO_TRY_EXIT_IF_NULL(
01197                 bad_pix_mask = cpl_image_duplicate(xcal));
01198 
01199             KMO_TRY_EXIT_IF_NULL(
01200                 pbad_pix_mask = cpl_image_get_data_float(bad_pix_mask));
01201             for (x = 0; x < nx; x++) {
01202                 for (y = 0; y < ny; y++) {
01203                     if (isnan(pbad_pix_mask[x+nx*y])) {
01204                         pbad_pix_mask[x+nx*y] = 0.;
01205                     } else {
01206                         pbad_pix_mask[x+nx*y] = 1.;
01207                     }
01208                 }
01209             }
01210             KMO_TRY_CHECK_ERROR_STATE();
01211 
01212             //
01213             // calculate SKYFLAT_EDGE
01214             //
01215             if (has_flat_edge) {
01216                 // get edge-edgepars from FLAT_SKY
01217                 KMO_TRY_EXIT_IF_ERROR(
01218                     kmo_calc_edgepars(combined_data,
01219                                       unused_ifus_after[det_nr-1],
01220                                       bad_pix_mask,
01221                                       det_nr,
01222                                       &slitlet_ids,
01223                                       &edgepars));
01224                 KMO_TRY_CHECK_ERROR_STATE();
01225 
01226                 // copy edgepars to table for saving later on
01227                 KMO_TRY_EXIT_IF_NULL(
01228                     edge_table_sky[det_nr-1] = kmo_edgepars_to_table(slitlet_ids, edgepars));
01229 
01230                 if (edgepars != NULL) {
01231                     for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01232                         cpl_matrix_delete(edgepars[i]); edgepars[i] = NULL;
01233                     }
01234                     cpl_free(edgepars); edgepars = NULL;
01235                 }
01236                 if (slitlet_ids != NULL) {
01237                     for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01238                         cpl_vector_delete(slitlet_ids[i]); slitlet_ids[i] = NULL;
01239                     }
01240                     cpl_free(slitlet_ids); slitlet_ids = NULL;
01241                 }
01242                 KMO_TRY_CHECK_ERROR_STATE();
01243 
01244                 //
01245                 // correlate FLAT_EDGE and SKYFLAT_EDGE
01246                 //
01247 
01248                 // load flat_edge from MASTER_FLAT
01249                 KMO_TRY_EXIT_IF_NULL(
01250                     frame = kmo_dfs_get_frame(frameset, FLAT_EDGE));
01251                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01252                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01253                     KMO_TRY_EXIT_IF_NULL(
01254                         edge_table_flat[j] = kmclipm_cal_table_load(cpl_frame_get_filename(frame),
01255                                                                     ifu_nr, rotangle, 0, &tmp_rotangle));
01256                 }
01257 
01258                 //
01259                 // calculate shift value
01260                 //
01261 
01262                 KMO_TRY_EXIT_IF_NULL(
01263                     shift_vec = cpl_vector_new(KMOS_IFUS_PER_DETECTOR));
01264                 KMO_TRY_EXIT_IF_NULL(
01265                     edge_vec = cpl_vector_new(2*KMOS_SLITLET_X));
01266 
01267                 // get shift values for each IFU by comparing all edge parameters,
01268                 // rejecting and applying median
01269                 int row = 1024; // middle of frame
01270                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01271                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01272                     for (edgeNr = 0; edgeNr < 2*KMOS_SLITLET_X; edgeNr++) {
01273                         double flatval = kmo_calc_fitted_slitlet_edge(edge_table_flat[j], edgeNr, row);
01274                         double skyval  = kmo_calc_fitted_slitlet_edge(edge_table_sky[det_nr-1][j], edgeNr, row);
01275                         cpl_vector_set(edge_vec, edgeNr, flatval-skyval);
01276                     }
01277 
01278                     // reject deviating edge-differences
01279                     kmclipm_vector *kv = NULL;
01280                     KMO_TRY_EXIT_IF_NULL(
01281                         kv = kmclipm_vector_create(cpl_vector_duplicate(edge_vec)));
01282                     kmclipm_reject_deviant(kv, 3, 3, NULL, NULL);
01283 
01284                     // set shift value for each IFU
01285                     cpl_vector_set(shift_vec, j, kmclipm_vector_get_median(kv, KMCLIPM_ARITHMETIC));
01286                     kmclipm_vector_delete(kv); kv = NULL;
01287                 }
01288                 cpl_vector_delete(edge_vec); edge_vec = NULL;
01289                 KMO_TRY_CHECK_ERROR_STATE();
01290 
01291                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01292                     cpl_table_delete(edge_table_flat[j]);
01293                     edge_table_flat[j] = NULL;
01294                 }
01295 
01296                 // take median of all IFU-shift-values
01297                 double shift_val = -cpl_vector_get_median(shift_vec);
01298                 cpl_vector_delete(shift_vec); shift_vec = NULL;
01299 
01300                 cpl_msg_info("", "Shift detector %d by %g pixels.", det_nr, shift_val);
01301 
01302                 int     xdim                = cpl_image_get_size_x(combined_data),
01303                         ydim                = cpl_image_get_size_y(combined_data);
01304                 double  *array_in           = cpl_calloc(xdim, sizeof(double)),
01305                         *array_out          = NULL;
01306                 float   *pcombined_data     = cpl_image_get_data_float(combined_data),
01307                         *pcombined_noise    = NULL;
01308     //            float   *tmpArray           = cpl_calloc(xdim, sizeof(float));
01309                 if (process_noise) {
01310                     pcombined_noise = cpl_image_get_data_float(combined_noise);
01311                 }
01312 
01313                 for (iy = 0; iy < ydim; iy++) {
01314                     // cubic spline
01315                     for (ix = 0; ix < xdim; ix++) {
01316                         array_in[ix] = pcombined_data[ix+iy*xdim];
01317                     }
01318                     array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01319                                                     xdim, shift_val, 1.0,
01320                                                     NATURAL);
01321                     for (ix = 0; ix < xdim; ix++) {
01322                       pcombined_data[ix+iy*xdim] = array_out[ix];
01323                     }
01324                     cpl_free(array_out);
01325 
01326     //                // linear
01327     //                for (ix = 1; ix < xdim; ix++) {
01328     //                    tmpArray[ix-1] = (pcombined_data[ix+iy*xdim]-pcombined_data[(ix-1)+iy*xdim])*shift_val +
01329     //                                     pcombined_data[(ix-1)+iy*xdim];
01330     //                }
01331     //                for (ix = 1; ix < xdim; ix++) {
01332     //                    pcombined_data[ix+iy*xdim] = tmpArray[ix];
01333     //                }
01334 
01335                     if (process_noise) {
01336                         // cubic spline
01337                         for (ix = 0; ix < xdim; ix++) {
01338                             array_in[ix] = pcombined_noise[ix+iy*xdim];
01339                         }
01340                         array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01341                                                         xdim, shift_val, 1.0,
01342                                                         NATURAL);
01343                         for (ix = 0; ix < xdim; ix++) {
01344                           pcombined_noise[ix+iy*xdim] = array_out[ix];
01345                         }
01346                         cpl_free(array_out);
01347 
01348     //                    // linear
01349     //                    for (ix = 1; ix < xdim; ix++) {
01350     //                        tmpArray[ix-1] = (pcombined_noise[ix+iy*xdim]-pcombined_noise[(ix-1)+iy*xdim])*shift_val +
01351     //                                         pcombined_noise[(ix-1)+iy*xdim];
01352     //                    }
01353     //                    for (ix = 1; ix < xdim; ix++) {
01354     //                        pcombined_noise[ix+iy*xdim] = tmpArray[ix];
01355     //                    }
01356                     }
01357                 }
01358                 cpl_free(array_in); array_in = NULL;
01359             }
01360             //
01361             // reconstruct
01362             //
01363             // load MASTER_DARK and MASTER_FLAT
01364             KMO_TRY_EXIT_IF_NULL(
01365                 img_dark = kmo_dfs_load_image(frameset, MASTER_DARK,
01366                                               det_nr, FALSE, FALSE, NULL));
01367 
01368             if (process_noise) {
01369                 KMO_TRY_EXIT_IF_NULL(
01370                     img_dark_noise = kmo_dfs_load_image(frameset, MASTER_DARK,
01371                                                         det_nr, TRUE, FALSE, NULL));
01372             }
01373 
01374             KMO_TRY_EXIT_IF_NULL(
01375                 img_flat = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, FALSE,
01376                                                   rotangle, FALSE, NULL,
01377                                                   &rotangle_found, -1, 0, 0));
01378 
01379             if (process_noise) {
01380                 KMO_TRY_EXIT_IF_NULL(
01381                     img_flat_noise = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, TRUE,
01382                                                             rotangle, FALSE, NULL,
01383                                                             &rotangle_found, -1, 0, 0));
01384             }
01385 
01386             char *tmp_band_method = getenv("KMO_BAND_METHOD");
01387             int band_method = 0;
01388             if (tmp_band_method != NULL) {
01389                 band_method = atoi(tmp_band_method);
01390             }
01391 
01392             // ESO INS FILTi ID
01393             KMO_TRY_EXIT_IF_NULL(
01394                 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, det_nr,
01395                                       IFU_FILTID_POSTFIX));
01396             KMO_TRY_EXIT_IF_NULL(
01397                 filter_id = cpl_propertylist_get_string(main_header, keyword));
01398             cpl_free(keyword); keyword = NULL;
01399 
01400             KMO_TRY_EXIT_IF_NULL(
01401                 band_table = kmo_dfs_load_table(frameset, WAVE_BAND, 1, 0));
01402             KMO_TRY_EXIT_IF_ERROR(
01403                 kmclipm_setup_grid_band_lcal(&gd, lcal, filter_id, band_method,
01404                                              band_table));
01405             cpl_table_delete(band_table); band_table = NULL;
01406 
01407             cpl_msg_info("","Reconstructing cubes...");
01408             for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01409                 // update sub-header
01410                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01411 
01412                 // load raw image and sub-header
01413                 KMO_TRY_EXIT_IF_NULL(
01414                     sub_header = kmo_dfs_load_sub_header(frameset_sky, FLAT_SKY,
01415                                                          det_nr, FALSE));
01416 
01417                 KMO_TRY_EXIT_IF_NULL(
01418                     punused_ifus = cpl_array_get_data_int_const(
01419                                                   unused_ifus_after[det_nr-1]));
01420 
01421                 // check if IFU is valid according to main header keywords &
01422                 // calibration files
01423                 KMO_TRY_EXIT_IF_NULL(
01424                     keyword = cpl_sprintf("%s%d%s", IFU_VALID_PREFIX, ifu_nr,
01425                                           IFU_VALID_POSTFIX));
01426                 KMO_TRY_CHECK_ERROR_STATE();
01427                 ranges_txt = cpl_propertylist_get_string(main_header, keyword);
01428                 cpl_free(keyword); keyword = NULL;
01429 
01430                 if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) &&
01431                     (bounds[2*(ifu_nr-1)] != -1) &&
01432                     (bounds[2*(ifu_nr-1)+1] != -1) &&
01433                     (punused_ifus[j] == 0))
01434                 {
01435                     // IFU is valid
01436                     cpl_error_reset();
01437 
01438                     // calculate WCS
01439                     KMO_TRY_EXIT_IF_ERROR(
01440                         kmo_calc_wcs_gd(main_header, sub_header, ifu_nr, gd));
01441 
01442                     // reconstruct data
01443                     KMO_TRY_EXIT_IF_ERROR(
01444                         kmo_reconstruct_sci_image(ifu_nr,
01445                                                 bounds[2*(ifu_nr-1)],
01446                                                 bounds[2*(ifu_nr-1)+1],
01447                                                 combined_data,
01448                                                 combined_noise,
01449                                                 img_dark,
01450                                                 img_dark_noise,
01451                                                 img_flat,
01452                                                 img_flat_noise,
01453                                                 xcal,
01454                                                 ycal,
01455                                                 lcal,
01456                                                 &gd,
01457                                                 calTimestamp,
01458                                                 calAngles,
01459                                                 fn_lut,
01460                                                 &cube_data,
01461                                                 &cube_noise,
01462                                                 flux,
01463                                                 background));
01464                     KMO_TRY_CHECK_ERROR_STATE();
01465                 } else {
01466                     // IFU is invalid
01467                     cpl_error_reset();
01468                 } // if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) ...
01469 
01470                 // save output
01471                 KMO_TRY_EXIT_IF_NULL(
01472                     extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_DATA));
01473 
01474                 KMO_TRY_EXIT_IF_ERROR(
01475                     kmclipm_update_property_string(sub_header, EXTNAME,
01476                                                    extname,
01477                                                    "FITS extension name"));
01478 
01479                 cpl_free(extname); extname = NULL;
01480 
01481                 // store cube and sub header into array for later
01482                 stored_data_cubes[ifu_nr - 1] = cube_data;
01483                 stored_sub_data_headers[ifu_nr - 1] = sub_header;
01484 
01485                 if (process_noise) {
01486                     KMO_TRY_EXIT_IF_NULL(
01487                         sub_header = cpl_propertylist_duplicate(
01488                                            stored_sub_data_headers[ifu_nr - 1]));
01489                     KMO_TRY_EXIT_IF_NULL(
01490                         extname = kmo_extname_creator(ifu_frame, ifu_nr,
01491                                                       EXT_NOISE));
01492 
01493                     KMO_TRY_EXIT_IF_ERROR(
01494                         kmclipm_update_property_string(sub_header,
01495                                                 EXTNAME,
01496                                                 extname,
01497                                                 "FITS extension name"));
01498 
01499                     cpl_free(extname); extname = NULL;
01500 
01501                     stored_noise_cubes[ifu_nr - 1] = cube_noise;
01502                     stored_sub_noise_headers[ifu_nr - 1] = sub_header;
01503                 }
01504                 cpl_image_delete(data_ifu); data_ifu = NULL;
01505                 cpl_image_delete(noise_ifu); noise_ifu = NULL;
01506                 cube_data = NULL;
01507                 cube_noise = NULL;
01508             } // for j IFUs
01509 
01510             // free memory
01511             cpl_image_delete(combined_data); combined_data = NULL;
01512             cpl_image_delete(combined_noise); combined_noise = NULL;
01513             cpl_image_delete(xcal); xcal = NULL;
01514             cpl_image_delete(ycal); ycal = NULL;
01515             cpl_image_delete(lcal); lcal = NULL;
01516             cpl_image_delete(img_dark); img_dark = NULL;
01517             cpl_image_delete(img_flat); img_flat = NULL;
01518             cpl_image_delete(bad_pix_mask); bad_pix_mask = NULL;
01519             if (process_noise) {
01520                 cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01521                 cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01522             }
01523         } // for nr_devices
01524 
01525         cpl_free(edge_table_flat); edge_table_flat = NULL;
01526 
01527         // collapse cubes using rejection
01528         cpl_msg_info("","Collapsing cubes...");
01529         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01530             for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01531                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01532 
01533                 KMO_TRY_EXIT_IF_NULL(
01534                     punused_ifus = cpl_array_get_data_int_const(
01535                                                   unused_ifus_after[det_nr-1]));
01536                 if (punused_ifus[j] == 0) {
01537                     if (stored_sub_data_headers[ifu_nr-1] != NULL) {
01538                     // IFU is valid
01539                     ifu_crpix = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01540                                                             CRPIX3);
01541                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01542                                    "CRPIX3 keyword in FITS-header is missing!");
01543 
01544                     ifu_crval = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01545                                                             CRVAL3);
01546                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01547                                    "CRVAL3 keyword in FITS-header is missing!");
01548 
01549                     ifu_cdelt = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01550                                                             CDELT3);
01551                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01552                                    "CDELT3 keyword in FITS-header is missing!");
01553 
01554                     KMO_TRY_EXIT_IF_NULL(
01555                         identified_slices = kmo_identify_slices(ranges,
01556                                                                 ifu_crpix,
01557                                                                 ifu_crval,
01558                                                                 ifu_cdelt,
01559                                                                 gd.l.dim));
01560                     }/* else {
01561                         KMO_TRY_EXIT_IF_NULL(
01562                             identified_slices = cpl_vector_new(gd.l.dim));
01563                         cpl_vector_fill(identified_slices, 1.);
01564                     }*/
01565 
01566                     if (stored_data_cubes[ifu_nr-1] != NULL) {
01567                         KMO_TRY_EXIT_IF_ERROR(
01568                             kmclipm_make_image(stored_data_cubes[ifu_nr-1],
01569                                                stored_noise_cubes[ifu_nr-1],
01570                                                &stored_data_images[ifu_nr-1],
01571                                                &stored_noise_images[ifu_nr-1],
01572                                                identified_slices,
01573                                                cmethod, cpos_rej, cneg_rej,
01574                                                citer, cmax, cmin));
01575                     }
01576                     cpl_vector_delete(identified_slices); identified_slices = NULL;
01577                 } else {
01578                     // IFU is invalid
01579                 }
01580             }
01581         }
01582 
01583         // normalise all IFUs as a group.
01584         // Calculate mean of each IFU, add up and divide by number of successful
01585         // averaged IFUs.
01586         // Then divide all valid IFUs with mean value
01587         cnt = 0;
01588         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01589             if (stored_data_images[i] != NULL) {
01590                 KMO_TRY_ASSURE(cpl_image_count_rejected(stored_data_images[i]) <
01591                                cpl_image_get_size_x(stored_data_images[i])*
01592                                cpl_image_get_size_y(stored_data_images[i]),
01593                                CPL_ERROR_ILLEGAL_INPUT,
01594                                "The collapsed, dark-subtracted image contains "
01595                                "only invalid values! Probably the provided "
01596                                "FLAT_SKY frames are exactly the same as the "
01597                                "frames used for MASTER_DARK calculation.");
01598 
01599                 mean_data += cpl_image_get_mean(stored_data_images[i]);
01600                 KMO_TRY_CHECK_ERROR_STATE();
01601                 cnt++;
01602             }
01603 
01604         }
01605         mean_data /= cnt;
01606 
01607         if (mean_data != 0.0) {
01608             for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01609                 if (stored_data_images[i] != NULL) {
01610                     KMO_TRY_EXIT_IF_ERROR(
01611                         cpl_image_divide_scalar(stored_data_images[i],
01612                                                 mean_data));
01613                 }
01614             }
01615         } else {
01616             cpl_msg_warning(cpl_func, "Data couldn't be normalised "
01617                                       "(mean = 0.0)!");
01618         }
01619 
01620         if (process_noise) {
01621             if (mean_data != 0.0) {
01622                 for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01623                     if (stored_noise_images[i] != NULL) {
01624                         KMO_TRY_EXIT_IF_ERROR(
01625                             cpl_image_divide_scalar(stored_noise_images[i],
01626                                                     mean_data));
01627                     }
01628                 }
01629             } else {
01630                 cpl_msg_warning(cpl_func, "Noise couldn't be normalised "
01631                                           "(mean = 0.0)!");
01632             }
01633         }
01634 
01635         // calculate qc parameters on normalised data
01636         qc_spat_unif = 0.0;
01637         cnt = 0;
01638         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01639             if (stored_data_images[i] != NULL) {
01640                 tmp_mean = cpl_image_get_mean(stored_data_images[i]);
01641                 tmp_stdev = cpl_image_get_stdev (stored_data_images[i]);
01642 
01643                 qc_spat_unif += pow(tmp_mean-1, 2);
01644                 if (fabs(tmp_mean) > qc_max_dev) {
01645                     qc_max_dev = tmp_mean-1;
01646                     qc_max_dev_id = i+1;
01647                 }
01648                 if (fabs(tmp_stdev) > qc_max_nonunif) {
01649                     qc_max_nonunif = tmp_stdev;
01650                     qc_max_nonunif_id = i+1;
01651                 }
01652                 KMO_TRY_CHECK_ERROR_STATE();
01653                 cnt++;
01654             }
01655         }
01656         qc_spat_unif = sqrt(qc_spat_unif / cnt);
01657 
01658         //
01659         // save data
01660         //
01661 
01662         // update which IFUs are not used
01663         kmo_print_unused_ifus(unused_ifus_after, TRUE);
01664 
01665         KMO_TRY_EXIT_IF_ERROR(
01666             kmo_set_unused_ifus(unused_ifus_after, main_header,
01667                                 "kmo_illumination"));
01668 
01669         cpl_msg_info("","Saving data...");
01670 
01671         KMO_TRY_EXIT_IF_ERROR(
01672             kmclipm_update_property_double(main_header, QC_SPAT_UNIF, qc_spat_unif,
01673                                            "[adu] uniformity of illumination correction"));
01674         KMO_TRY_EXIT_IF_ERROR(
01675             kmclipm_update_property_double(main_header, QC_SPAT_MAX_DEV, qc_max_dev,
01676                                            "[adu] max. deviation from unity"));
01677         KMO_TRY_EXIT_IF_ERROR(
01678             kmclipm_update_property_int(main_header, QC_SPAT_MAX_DEV_ID, qc_max_dev_id,
01679                                         "[] IFU ID with max. dev. from unity"));
01680         KMO_TRY_EXIT_IF_ERROR(
01681             kmclipm_update_property_double(main_header, QC_SPAT_MAX_NONUNIF, qc_max_nonunif,
01682                                            "[adu] max. stdev of illumination corr."));
01683         KMO_TRY_EXIT_IF_ERROR(
01684             kmclipm_update_property_int(main_header, QC_SPAT_MAX_NONUNIF_ID, qc_max_nonunif_id,
01685                                         "[] IFU ID with max. stdev in illum. corr."));
01686 
01687         if (!suppress_extension) {
01688             KMO_TRY_EXIT_IF_NULL(
01689                 fn_suffix = cpl_sprintf("%s", suffix));
01690         } else {
01691             KMO_TRY_EXIT_IF_NULL(
01692                 fn_suffix = cpl_sprintf("%s", ""));
01693         }
01694         KMO_TRY_EXIT_IF_ERROR(
01695             kmo_dfs_save_main_header(frameset, ILLUM_CORR, fn_suffix, frame,
01696                                      main_header, parlist, cpl_func));
01697 
01698         KMO_TRY_EXIT_IF_ERROR(
01699             kmo_dfs_save_main_header(frameset, SKYFLAT_EDGE, fn_suffix, frame,
01700                                      main_header, parlist, cpl_func));
01701 
01702         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01703             KMO_TRY_EXIT_IF_ERROR(
01704                 kmo_dfs_save_image(stored_data_images[i], ILLUM_CORR, fn_suffix,
01705                                    stored_sub_data_headers[i], 0./0.));
01706 
01707             if (process_noise) {
01708                 KMO_TRY_EXIT_IF_ERROR(
01709                     kmo_dfs_save_image(stored_noise_images[i], ILLUM_CORR,
01710                                        fn_suffix, stored_sub_noise_headers[i], 0./0.));
01711             }
01712         }
01713 
01714         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01715             for (ifu_nr = 0; ifu_nr < KMOS_IFUS_PER_DETECTOR; ifu_nr++) {
01716                 KMO_TRY_EXIT_IF_ERROR(
01717                     kmclipm_update_property_int(stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01718                                                 CAL_IFU_NR,
01719                                                 ifu_nr+1+(det_nr-1)*KMOS_IFUS_PER_DETECTOR,
01720                                                 "IFU Number {1..24}"));
01721                 KMO_TRY_EXIT_IF_ERROR(
01722                     kmclipm_update_property_double(
01723                                                 stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01724                                                 CAL_ROTANGLE,
01725                                                 rotangle_found,
01726                                                 "[deg] Rotator relative to nasmyth"));
01727                 if (has_flat_edge) {
01728                     // save edge-parameters as product
01729                     KMO_TRY_EXIT_IF_ERROR(
01730                         kmo_dfs_save_table(edge_table_sky[det_nr-1][ifu_nr], SKYFLAT_EDGE, fn_suffix,
01731                                            stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr]));
01732                 }
01733             }
01734         }
01735     }
01736     KMO_CATCH
01737     {
01738         KMO_CATCH_MSG();
01739         ret_val = -1;
01740     }
01741     kmo_free_fits_desc(&desc_sky);
01742     kmo_free_fits_desc(&desc_dark);
01743     kmo_free_fits_desc(&desc_flat);
01744     kmo_free_fits_desc(&desc_xcal);
01745     kmo_free_fits_desc(&desc_ycal);
01746     kmo_free_fits_desc(&desc_lcal);
01747     cpl_image_delete(combined_data); combined_data = NULL;
01748     cpl_image_delete(combined_noise); combined_noise = NULL;
01749     cpl_image_delete(xcal); xcal = NULL;
01750     cpl_image_delete(ycal); ycal = NULL;
01751     cpl_image_delete(lcal); lcal = NULL;
01752     cpl_image_delete(img_dark); img_dark = NULL;
01753     cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01754     cpl_image_delete(img_flat); img_flat = NULL;
01755     cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01756     cpl_array_delete(calTimestamp); calTimestamp = NULL;
01757     cpl_free(bounds); bounds = NULL;
01758     kmo_free_unused_ifus(unused_ifus_before); unused_ifus_before = NULL;
01759     kmo_free_unused_ifus(unused_ifus_after); unused_ifus_after = NULL;
01760     cpl_free(fn_lut); fn_lut = NULL;
01761     cpl_free(suffix); suffix = NULL;
01762     cpl_free(fn_suffix); fn_suffix = NULL;
01763     cpl_frameset_delete(frameset_sky); frameset_sky = NULL;
01764     cpl_vector_delete(ranges); ranges = NULL;
01765     cpl_free(filter); filter = NULL;
01766     if (calAngles != NULL) {
01767         cpl_vector_delete(calAngles); calAngles = NULL;
01768     }
01769     cpl_propertylist_delete(main_header); main_header = NULL;
01770     for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01771         if (stored_data_cubes != NULL) {
01772             cpl_imagelist_delete(stored_data_cubes[i]);
01773             stored_data_cubes[i] = NULL;
01774         }
01775         if (stored_noise_cubes != NULL) {
01776             cpl_imagelist_delete(stored_noise_cubes[i]);
01777             stored_noise_cubes[i] = NULL;
01778         }
01779         if (stored_data_images != NULL) {
01780             cpl_image_delete(stored_data_images[i]);
01781             stored_data_images[i] = NULL;
01782         }
01783         if (stored_noise_images != NULL) {
01784             cpl_image_delete(stored_noise_images[i]);
01785             stored_noise_images[i] = NULL;
01786         }
01787         if (stored_sub_data_headers != NULL) {
01788             cpl_propertylist_delete(stored_sub_data_headers[i]);
01789             stored_sub_data_headers[i] = NULL;
01790         }
01791         if (stored_sub_noise_headers != NULL) {
01792             cpl_propertylist_delete(stored_sub_noise_headers[i]);
01793             stored_sub_noise_headers[i] = NULL;
01794         }
01795     }
01796     cpl_free(stored_data_cubes); stored_data_cubes = NULL;
01797     cpl_free(stored_noise_cubes); stored_noise_cubes = NULL;
01798     cpl_free(stored_data_images); stored_data_images = NULL;
01799     cpl_free(stored_noise_images); stored_noise_images = NULL;
01800     cpl_free(stored_sub_data_headers); stored_sub_data_headers = NULL;
01801     cpl_free(stored_sub_noise_headers); stored_sub_noise_headers = NULL;
01802     if (edge_table_sky != NULL) {
01803         for (i = 0; i < KMOS_NR_DETECTORS; i++) {
01804             if (edge_table_sky[i] != NULL) {
01805                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01806                     cpl_table_delete(edge_table_sky[i][j]);
01807                     edge_table_sky[i][j] = NULL;
01808                 }
01809                 cpl_free(edge_table_sky[i]); edge_table_sky[i] = NULL;
01810             }
01811         }
01812         cpl_free(edge_table_sky); edge_table_sky = NULL;
01813     }
01814     if (edge_table_flat != NULL) {
01815         for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01816             cpl_table_delete(edge_table_flat[j]);
01817             edge_table_flat[j] = NULL;
01818         }
01819         cpl_free(edge_table_flat); edge_table_flat = NULL;
01820     }
01821 
01822     return ret_val;
01823 }
01824