KMOS Pipeline Reference Manual  1.1.3
kmo_illumination.c
00001 /* $Id: kmo_illumination.c,v 1.47 2013/05/13 12:09:14 aagudo Exp $
00002  *
00003  * This file is part of the KMOS Pipeline
00004  * Copyright (C) 2002,2003 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: aagudo $
00023  * $Date: 2013/05/13 12:09:14 $
00024  * $Revision: 1.47 $
00025  * $Name: HEAD $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 /*-----------------------------------------------------------------------------
00033  *                              Includes
00034  *----------------------------------------------------------------------------*/
00035 
00036 #include <math.h>
00037 #include <string.h>
00038 
00039 #include <cpl.h>
00040 
00041 #include "kmo_priv_reconstruct.h"
00042 #include "kmo_priv_functions.h"
00043 #include "kmo_priv_flat.h"
00044 #include "kmo_priv_wave_cal.h"
00045 #include "kmclipm_priv_splines.h"
00046 #include "kmo_functions.h"
00047 #include "kmo_cpl_extensions.h"
00048 #include "kmo_dfs.h"
00049 #include "kmo_error.h"
00050 #include "kmo_constants.h"
00051 #include "kmo_debug.h"
00052 
00053 /*-----------------------------------------------------------------------------
00054  *                          Functions prototypes
00055  *----------------------------------------------------------------------------*/
00056 
00057 static int kmo_illumination_create(cpl_plugin *);
00058 static int kmo_illumination_exec(cpl_plugin *);
00059 static int kmo_illumination_destroy(cpl_plugin *);
00060 static int kmo_illumination(cpl_parameterlist *, cpl_frameset *);
00061 
00062 /*-----------------------------------------------------------------------------
00063  *                          Static variables
00064  *----------------------------------------------------------------------------*/
00065 
00066 static char kmo_illumination_description[] =
00067 "This recipe creates the spatial non-uniformity calibration frame needed for\n"
00068 "all three detectors. It must be called after the kmo_wave_cal-recipe, which\n"
00069 "generates the spectral calibration frame needed in this recipe. As input at\n"
00070 "least a sky, a master dark, a master flat and the spatial and spectral cali-\n"
00071 "bration frames are required.\n"
00072 "\n"
00073 "BASIC PARAMETERS:\n"
00074 "-----------------\n"
00075 "--imethod\n"
00076 "The interpolation method used for reconstruction.\n"
00077 "\n"
00078 "--range\n"
00079 "The spectral ranges to combine when collapsing the reconstructed cubes. e.g.\n"
00080 "\"x1_start,x1_end;x2_start,x2_end\" (microns)\n"
00081 "\n"
00082 "ADVANCED PARAMETERS\n"
00083 "-------------------\n"
00084 "--add-all\n"
00085 "By default the first FLAT_SKY frame is omitted, since in the\n"
00086 "KMOS_spec_cal_skyflat template this is an acquisition frame to estimate the\n"
00087 "needed exposure time for the subsequent FLAT_SKY frames. If anyway all frames\n"
00088 "should be considered, set this parameter to TRUE.\n"
00089 "\n"
00090 "--neighborhoodRange\n"
00091 "Defines the range to search for neighbors during reconstruction\n"
00092 "\n"
00093 "--b_samples\n"
00094 "The number of samples in spectral direction for the reconstructed cube.\n"
00095 "Ideally this number should be greater than 2048, the detector size.\n"
00096 "\n"
00097 "--b_start\n"
00098 "--b_end\n"
00099 "Used to define manually the start and end wavelength for the reconstructed\n"
00100 "cube. By default the internally defined values are used.\n"
00101 "\n"
00102 "--cmethod\n"
00103 "Following methods of frame combination are available:\n"
00104 "   * 'ksigma' (Default)\n"
00105 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00106 "   are examined. If they deviate significantly, they will be rejected according\n"
00107 "   to the conditions:\n"
00108 "       val > mean + stdev * cpos_rej\n"
00109 "   and\n"
00110 "       val < mean - stdev * cneg_rej\n"
00111 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00112 "   parameters. In the first iteration median and percentile level are used.\n"
00113 "\n"
00114 "   * 'median'\n"
00115 "   At each pixel position the median is calculated.\n"
00116 "\n"
00117 "   * 'average'\n"
00118 "   At each pixel position the average is calculated.\n"
00119 "\n"
00120 "   * 'sum'\n"
00121 "   At each pixel position the sum is calculated.\n"
00122 "\n"
00123 "   * 'min_max'\n"
00124 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00125 "   --cmax and --cmin apply to this method.\n"
00126 "\n"
00127 "--cpos_rej\n"
00128 "--cneg_rej\n"
00129 "--citer\n"
00130 "see --cmethod='ksigma'\n"
00131 "\n"
00132 "--cmax\n"
00133 "--cmin\n"
00134 "see --cmethod='min_max'\n"
00135 "\n"
00136 "--pix_scale"
00137 "Change the pixel scale [arcsec]. Default of 0.2\" results into cubes of\n"
00138 "14x14pix, a scale of 0.1\" results into cubes of 28x28pix, etc.\n"
00139 "\n"
00140 "-------------------------------------------------------------------------------\n"
00141 "  Input files:\n"
00142 "\n"
00143 "   DO                    KMOS                                                  \n"
00144 "   category              Type   Explanation                    Required #Frames\n"
00145 "   --------              -----  -----------                    -------- -------\n"
00146 "   FLAT_SKY               F2D   Sky exposures                     Y      1-n   \n"
00147 "                                (at least 3 frames recommended)                \n"
00148 "   MASTER_DARK            F2D   Master dark                       Y       1    \n"
00149 "   MASTER_FLAT            F2D   Master flat                       Y       1    \n"
00150 "   XCAL                   F2D   x calibration frame               Y       1    \n"
00151 "   YCAL                   F2D   y calibration frame               Y       1    \n"
00152 "   LCAL                   F2D   Wavelength calib. frame           Y       1    \n"
00153 "   WAVE_BAND              F2L   Table with start-/end-wavelengths Y       1    \n"
00154 "   FLAT_EDGE              F2L   Table with fitted slitlet edges   N      0,1   \n"
00155 "\n"
00156 "  Output files:\n"
00157 "\n"
00158 "   DO                    KMOS\n"
00159 "   category              Type   Explanation\n"
00160 "   --------              -----  -----------\n"
00161 "   ILLUM_CORR            F2I    Illumination calibration frame   \n"
00162 "   If FLAT_EDGE is provided: \n"
00163 "   SKYFLAT_EDGE          F2L    Frame containing parameters of fitted \n"
00164 "                                slitlets of all IFUs of all detectors\n"
00165 "-------------------------------------------------------------------------------\n"
00166 "\n";
00167 
00168 /*-----------------------------------------------------------------------------
00169  *                              Functions code
00170  *----------------------------------------------------------------------------*/
00171 
00188 int cpl_plugin_get_info(cpl_pluginlist *list)
00189 {
00190     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00191     cpl_plugin *plugin = &recipe->interface;
00192 
00193     cpl_plugin_init(plugin,
00194                         CPL_PLUGIN_API,
00195                         KMOS_BINARY_VERSION,
00196                         CPL_PLUGIN_TYPE_RECIPE,
00197                         "kmo_illumination",
00198                         "Create a calibration frame to correct spatial "
00199                         "non-uniformity of flatfield.",
00200                         kmo_illumination_description,
00201                         "Alex Agudo Berbel",
00202                         "agudo@mpe.mpg.de",
00203                         kmos_get_license(),
00204                         kmo_illumination_create,
00205                         kmo_illumination_exec,
00206                         kmo_illumination_destroy);
00207 
00208     cpl_pluginlist_append(list, plugin);
00209 
00210     return 0;
00211 }
00212 
00220 static int kmo_illumination_create(cpl_plugin *plugin)
00221 {
00222     cpl_recipe *recipe;
00223     cpl_parameter *p;
00224 
00225     /* Check that the plugin is part of a valid recipe */
00226     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00227         recipe = (cpl_recipe *)plugin;
00228     else
00229         return -1;
00230 
00231     /* Create the parameters list in the cpl_recipe object */
00232     recipe->parameters = cpl_parameterlist_new();
00233 
00234     /* Fill the parameters list */
00235     /* --imethod */
00236     p = cpl_parameter_new_value("kmos.kmo_illumination.imethod",
00237                                 CPL_TYPE_STRING,
00238                                 "Method to use for interpolation: "
00239                                 "[\"NN\" (nearest neighbour), "
00240                                 "\"lwNN\" (linear weighted nearest neighbor),"
00241                                 "\"swNN\" (square weighted nearest neighbor)"
00242                                 "\"MS\" (Modified Shepard's method)"
00243                                 "\"CS\" (Cubic spline)]",
00244                                 "kmos.kmo_illumination",
00245                                 "CS");
00246     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "imethod");
00247     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00248     cpl_parameterlist_append(recipe->parameters, p);
00249 
00250     /* --neighborhoodRange */
00251     p = cpl_parameter_new_value("kmos.kmo_illumination.neighborhoodRange",
00252                                 CPL_TYPE_DOUBLE,
00253                                 "Defines the range to search for neighbors."
00254                                 "in pixels",
00255                                 "kmos.kmo_illumination",
00256                                 1.001);
00257     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "neighborhoodRange");
00258     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00259     cpl_parameterlist_append(recipe->parameters, p);
00260 
00261     /* --range */
00262     p = cpl_parameter_new_value("kmos.kmo_illumination.range",
00263                                 CPL_TYPE_STRING,
00264                                 "The spectral ranges to combine when collapsing"
00265                                 "the reconstructed cubes. e.g."
00266                                 "\"x1_start,x1_end;x2_start,x2_end\" (microns)",
00267                                 "kmos.kmo_illumination",
00268                                 "");
00269     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "range");
00270     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00271     cpl_parameterlist_append(recipe->parameters, p);
00272 
00273     /* --flux */
00274     p = cpl_parameter_new_value("kmos.kmo_illumination.flux",
00275                                 CPL_TYPE_BOOL,
00276                                 "TRUE: Apply flux conservation. FALSE: otherwise",
00277                                 "kmos.kmo_illumination",
00278                                 FALSE);
00279     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00280     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00281     cpl_parameterlist_append(recipe->parameters, p);
00282 
00283     /* --add-all */
00284     p = cpl_parameter_new_value("kmos.kmo_illumination.add-all",
00285                                 CPL_TYPE_BOOL,
00286                                 "FALSE: omit 1st FLAT_SKY frame (acquisition), "
00287                                 "TRUE: don't perform any checks, add them all",
00288                                 "kmos.kmo_illumination",
00289                                 FALSE);
00290     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "add-all");
00291     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00292     cpl_parameterlist_append(recipe->parameters, p);
00293 
00294     /* --pix_scale */
00295     p = cpl_parameter_new_value("kmos.kmo_illumination.pix_scale",
00296                                 CPL_TYPE_DOUBLE,
00297                                 "Change the pixel scale [arcsec]. "
00298                                 "Default of 0.2\" results into cubes of 14x14pix, "
00299                                 "a scale of 0.1\" results into cubes of 28x28pix, "
00300                                 "etc.",
00301                                 "kmos.kmo_illumination",
00302                                 KMOS_PIX_RESOLUTION);
00303     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pix_scale");
00304     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00305     cpl_parameterlist_append(recipe->parameters, p);
00306 
00307     // add parameters for band-definition
00308     kmo_band_pars_create(recipe->parameters,
00309                          "kmos.kmo_illumination");
00310 
00311     // add parameters for combining
00312     return kmo_combine_pars_create(recipe->parameters,
00313                                    "kmos.kmo_illumination",
00314                                    DEF_REJ_METHOD,
00315                                    FALSE);
00316 }
00317 
00323 static int kmo_illumination_exec(cpl_plugin *plugin)
00324 {
00325     cpl_recipe  *recipe;
00326 
00327     /* Get the recipe out of the plugin */
00328     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00329         recipe = (cpl_recipe *)plugin;
00330     else return -1;
00331 
00332     return kmo_illumination(recipe->parameters, recipe->frames);
00333 }
00334 
00340 static int kmo_illumination_destroy(cpl_plugin *plugin)
00341 {
00342     cpl_recipe *recipe;
00343 
00344     /* Get the recipe out of the plugin */
00345     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00346         recipe = (cpl_recipe *)plugin;
00347     else return -1 ;
00348 
00349     cpl_parameterlist_delete(recipe->parameters);
00350     return 0 ;
00351 }
00352 
00367 static int kmo_illumination(cpl_parameterlist *parlist, cpl_frameset *frameset)
00368 {
00369     int              ret_val                    = 0,
00370                      nr_devices                 = 0,
00371                      ifu_nr                     = 0,
00372                      nx                         = 0,
00373                      ny                         = 0,
00374                      process_noise              = FALSE,
00375                      cmax                       = 0,
00376                      cmin                       = 0,
00377                      citer                      = 0,
00378                      *bounds                    = NULL,
00379                      cnt                        = 0,
00380                      qc_max_dev_id              = 0,
00381                      qc_max_nonunif_id          = 0,
00382                      flux                       = FALSE,
00383                      background                 = FALSE,
00384                      add_all_sky                = FALSE,
00385                      same_exptime               = TRUE,
00386                      has_flat_edge              = FALSE;
00387     const int        *punused_ifus              = NULL;
00388     float            *pbad_pix_mask             = NULL;
00389     double           exptime                    = 0.,
00390                      exptime1                   = 0.,
00391                      exptime2                   = 0.,
00392                      cpos_rej                   = 0.0,
00393                      cneg_rej                   = 0.0,
00394                      neighborhoodRange          = 1.001,
00395                      mean_data                  = 0.0,
00396                      ifu_crpix                  = 0.0,
00397                      ifu_crval                  = 0.0,
00398                      ifu_cdelt                  = 0.0,
00399                      qc_spat_unif               = 0.0,
00400                      qc_max_dev                 = 0.0,
00401                      qc_max_nonunif             = 0.0,
00402                      tmp_stdev                  = 0.0,
00403                      tmp_mean                   = 0.0,
00404                      rotangle                   = 0.0,
00405                      tmp_rotangle               = 0.0,
00406                      rotangle_found             = 0.0,
00407                      pix_scale                  = 0.0;
00408     char             *keyword                   = NULL,
00409                      *fn_lut                    = NULL,
00410                      *suffix                    = NULL,
00411                      *extname                   = NULL,
00412                      *filter                    = NULL;
00413     const char       *method                    = NULL,
00414                      *cmethod                   = NULL,
00415                      *filter_id_l               = NULL,
00416                      *filter_id                 = NULL,
00417                      *ranges_txt                = NULL;
00418     cpl_array        *calTimestamp              = NULL,
00419                      **unused_ifus_before       = NULL,
00420                      **unused_ifus_after        = NULL;
00421     cpl_frame        *frame                     = NULL,
00422                      *xcalFrame                 = NULL,
00423                      *ycalFrame                 = NULL,
00424                      *lcalFrame                 = NULL;
00425     cpl_frameset     *frameset_sky              = NULL;
00426     cpl_image        *img_in                    = NULL,
00427                      *img_dark                  = NULL,
00428                      *img_dark_noise            = NULL,
00429                      *img_flat                  = NULL,
00430                      *img_flat_noise            = NULL,
00431                      *combined_data             = NULL,
00432                      *combined_noise            = NULL,
00433                      *xcal                      = NULL,
00434                      *ycal                      = NULL,
00435                      *lcal                      = NULL,
00436                      *bad_pix_mask              = NULL,
00437                      *data_ifu                  = NULL,
00438                      *noise_ifu                 = NULL,
00439                      **stored_data_images       = NULL,
00440                      **stored_noise_images      = NULL;
00441     cpl_imagelist    *cube_data                 = NULL,
00442                      *cube_noise                = NULL,
00443                      *detector_in               = NULL,
00444                      **stored_data_cubes        = NULL,
00445                      **stored_noise_cubes       = NULL;
00446     cpl_matrix       **edgepars                 = NULL;
00447     cpl_propertylist *main_header               = NULL,
00448                      *tmp_header                = NULL,
00449                      *sub_header                = NULL,
00450                      **stored_sub_data_headers  = NULL,
00451                      **stored_sub_noise_headers = NULL;
00452     cpl_table        *band_table                = NULL,
00453                      ***edge_table_sky          = NULL,
00454                      **edge_table_flat          = NULL;
00455     cpl_vector       *ranges                    = NULL,
00456                      *identified_slices         = NULL,
00457                      *calAngles                 = NULL,
00458                      **slitlet_ids              = NULL,
00459                      *shift_vec                 = NULL,
00460                      *edge_vec                  = NULL;
00461     main_fits_desc   desc_sky,
00462                      desc_dark,
00463                      desc_flat,
00464                      desc_xcal,
00465                      desc_ycal,
00466                      desc_lcal;
00467     gridDefinition   gd;
00468 
00469     KMO_TRY
00470     {
00471         kmo_init_fits_desc(&desc_sky);
00472         kmo_init_fits_desc(&desc_dark);
00473         kmo_init_fits_desc(&desc_flat);
00474         kmo_init_fits_desc(&desc_xcal);
00475         kmo_init_fits_desc(&desc_ycal);
00476         kmo_init_fits_desc(&desc_lcal);
00477 
00478         /* --- check input --- */
00479         KMO_TRY_ASSURE((parlist != NULL) &&
00480                        (frameset != NULL),
00481                        CPL_ERROR_NULL_INPUT,
00482                        "Not all input data is provided!");
00483 
00484         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, FLAT_SKY) >= 1,
00485                        CPL_ERROR_ILLEGAL_INPUT,
00486                        "One or more FLAT_SKY frames are required!");
00487 
00488         if (cpl_frameset_count_tags(frameset, FLAT_SKY) < 3) {
00489             cpl_msg_warning(cpl_func, "It is recommended to provide at least "
00490                                       "3 FLAT_SKY frames!");
00491         }
00492 
00493         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_DARK) == 1,
00494                        CPL_ERROR_ILLEGAL_INPUT,
00495                        "Exactly one MASTER_DARK frame is required!");
00496 
00497         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_FLAT) == 1,
00498                        CPL_ERROR_ILLEGAL_INPUT,
00499                        "Exactly one MASTER_FLAT frame is required!");
00500 
00501         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, XCAL) == 1,
00502                        CPL_ERROR_ILLEGAL_INPUT,
00503                        "Exactly one XCAL frame is required!");
00504 
00505         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, YCAL) == 1,
00506                        CPL_ERROR_ILLEGAL_INPUT,
00507                        "Exactly one YCAL frame is required!");
00508 
00509         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, LCAL) == 1,
00510                        CPL_ERROR_ILLEGAL_INPUT,
00511                        "Exactly one LCAL frame is required!");
00512 
00513         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, WAVE_BAND) == 1,
00514                        CPL_ERROR_ILLEGAL_INPUT,
00515                        "Exactly one WAVE_BAND frame is required!");
00516 
00517         KMO_TRY_ASSURE((cpl_frameset_count_tags(frameset, FLAT_EDGE) == 1) ||
00518                        (cpl_frameset_count_tags(frameset, FLAT_EDGE) == 0),
00519                        CPL_ERROR_ILLEGAL_INPUT,
00520                        "Exactly one FLAT_EDGE frame is required!");
00521 
00522         has_flat_edge = cpl_frameset_count_tags(frameset, FLAT_EDGE);
00523 
00524         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_illumination") == 1,
00525                        CPL_ERROR_ILLEGAL_INPUT,
00526                        "Cannot identify RAW and CALIB frames!");
00527 
00528         /* --- get parameters --- */
00529         cpl_msg_info("", "--- Parameter setup for kmo_illumination ---");
00530 
00531         KMO_TRY_EXIT_IF_NULL(
00532             method = kmo_dfs_get_parameter_string(parlist,
00533                                               "kmos.kmo_illumination.imethod"));
00534 
00535         KMO_TRY_ASSURE((strcmp(method, "NN") == 0) ||
00536                        (strcmp(method, "lwNN") == 0) ||
00537                        (strcmp(method, "swNN") == 0) ||
00538                        (strcmp(method, "MS") == 0) ||
00539                        (strcmp(method, "CS") == 0),
00540                        CPL_ERROR_ILLEGAL_INPUT,
00541                        "method must be either \"NN\", \"lwNN\", "
00542                        "\"swNN\", \"MS\" or \"CS\"!");
00543 
00544         KMO_TRY_EXIT_IF_ERROR(
00545             kmo_dfs_print_parameter_help(parlist,
00546                                         "kmos.kmo_illumination.imethod"));
00547 
00548         neighborhoodRange = kmo_dfs_get_parameter_double(parlist,
00549                 "kmos.kmo_illumination.neighborhoodRange");
00550         KMO_TRY_CHECK_ERROR_STATE();
00551 
00552         KMO_TRY_ASSURE(neighborhoodRange > 0.0,
00553                 CPL_ERROR_ILLEGAL_INPUT,
00554                 "neighborhoodRange must be greater than 0.0");
00555 
00556         KMO_TRY_EXIT_IF_ERROR(
00557             kmo_dfs_print_parameter_help(parlist,
00558                                     "kmos.kmo_illumination.neighborhoodRange"));
00559 
00560         ranges_txt = kmo_dfs_get_parameter_string(parlist,
00561                                                   "kmos.kmo_illumination.range");
00562         KMO_TRY_CHECK_ERROR_STATE();
00563 
00564         KMO_TRY_EXIT_IF_ERROR(
00565             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.range"));
00566 
00567         ranges = kmo_identify_ranges(ranges_txt);
00568         KMO_TRY_CHECK_ERROR_STATE();
00569 
00570         flux = kmo_dfs_get_parameter_bool(parlist,
00571                                           "kmos.kmo_illumination.flux");
00572 
00573         KMO_TRY_ASSURE((flux == 0) ||
00574                        (flux == 1),
00575                        CPL_ERROR_ILLEGAL_INPUT,
00576                        "flux must be either FALSE or TRUE!");
00577 
00578         KMO_TRY_EXIT_IF_ERROR(
00579             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.flux"));
00580 
00581         add_all_sky = kmo_dfs_get_parameter_bool(parlist,
00582                                                  "kmos.kmo_illumination.add-all");
00583 
00584         KMO_TRY_ASSURE((add_all_sky == 0) ||
00585                        (add_all_sky == 1),
00586                        CPL_ERROR_ILLEGAL_INPUT,
00587                        "add_all must be either FALSE or TRUE!");
00588 
00589         KMO_TRY_EXIT_IF_ERROR(
00590             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.add-all"));
00591 
00592         pix_scale = kmo_dfs_get_parameter_double(parlist,
00593                                         "kmos.kmo_illumination.pix_scale");
00594         KMO_TRY_CHECK_ERROR_STATE();
00595         KMO_TRY_EXIT_IF_ERROR(
00596            kmo_dfs_print_parameter_help(parlist,
00597                                        "kmos.kmo_illumination.pix_scale"));
00598         KMO_TRY_ASSURE((pix_scale >= 0.01) &&
00599                        (pix_scale <= 0.4),
00600                        CPL_ERROR_ILLEGAL_INPUT,
00601                        "pix_scale must be between 0.01 and 0.4 (results in cubes "
00602                        "with 7x7 to 280x280 pixels)!");
00603 
00604         kmo_band_pars_load(parlist, "kmos.kmo_illumination");
00605 
00606         KMO_TRY_EXIT_IF_ERROR(
00607             kmo_combine_pars_load(parlist,
00608                                   "kmos.kmo_illumination",
00609                                   &cmethod,
00610                                   &cpos_rej,
00611                                   &cneg_rej,
00612                                   &citer,
00613                                   &cmin,
00614                                   &cmax,
00615                                   FALSE));
00616         cpl_msg_info("", "-------------------------------------------");
00617 
00618         // check if filter_id, grating_id and rotator offset match for all
00619         // detectors
00620         KMO_TRY_EXIT_IF_ERROR(
00621             kmo_check_frameset_setup(frameset, FLAT_SKY,
00622                                        TRUE, FALSE, TRUE));
00623         KMO_TRY_EXIT_IF_ERROR(
00624             kmo_check_frame_setup(frameset, FLAT_SKY, XCAL,
00625                                        TRUE, FALSE, TRUE));
00626         KMO_TRY_EXIT_IF_ERROR(
00627             kmo_check_frame_setup(frameset, XCAL, YCAL,
00628                                        TRUE, FALSE, TRUE));
00629         KMO_TRY_EXIT_IF_ERROR(
00630             kmo_check_frame_setup(frameset, XCAL, LCAL,
00631                                        TRUE, FALSE, TRUE));
00632         KMO_TRY_EXIT_IF_ERROR(
00633             kmo_check_frame_setup(frameset, XCAL, MASTER_FLAT,
00634                                        TRUE, FALSE, TRUE));
00635 
00636         KMO_TRY_EXIT_IF_NULL(
00637             frame = kmo_dfs_get_frame(frameset, XCAL));
00638         KMO_TRY_EXIT_IF_NULL(
00639             suffix = kmo_dfs_get_suffix(frame, TRUE, FALSE));
00640 
00641         KMO_TRY_EXIT_IF_ERROR(
00642             kmo_check_frame_setup_md5_xycal(frameset));
00643         KMO_TRY_EXIT_IF_ERROR(
00644             kmo_check_frame_setup_md5(frameset));
00645 
00646         cpl_msg_info("", "Detected instrument setup:   %s", suffix+1);
00647         cpl_msg_info("", "(grating 1, 2 & 3)");
00648 
00649         // check which IFUs are active for all frames
00650         KMO_TRY_EXIT_IF_NULL(
00651             unused_ifus_before = kmo_get_unused_ifus(frameset, 0, 0));
00652 
00653         KMO_TRY_EXIT_IF_NULL(
00654             unused_ifus_after = kmo_duplicate_unused_ifus(unused_ifus_before));
00655 
00656         kmo_print_unused_ifus(unused_ifus_before, FALSE);
00657 
00658         // load desc for MASTER_DARK and check
00659         KMO_TRY_EXIT_IF_NULL(
00660             frame = kmo_dfs_get_frame(frameset, MASTER_DARK));
00661         desc_dark = kmo_identify_fits_header(
00662                     cpl_frame_get_filename(frame));
00663         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_DARK frame doesn't seem to "
00664                                       "be in KMOS-format!");
00665         KMO_TRY_ASSURE((desc_dark.nr_ext == 6) &&
00666                        (desc_dark.ex_badpix == FALSE) &&
00667                        (desc_dark.fits_type == f2d_fits) &&
00668                        (desc_dark.frame_type == detector_frame),
00669                        CPL_ERROR_ILLEGAL_INPUT,
00670                        "MASTER_DARK isn't in the correct format!!!");
00671         nx = desc_dark.naxis1;
00672         ny = desc_dark.naxis2;
00673 
00674         // load desc for MASTER_FLAT and check
00675         KMO_TRY_EXIT_IF_NULL(
00676             frame = kmo_dfs_get_frame(frameset, MASTER_FLAT));
00677         desc_flat = kmo_identify_fits_header(cpl_frame_get_filename(frame));
00678         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_FLAT frame doesn't seem to "
00679                                       "be in KMOS-format!");
00680         KMO_TRY_ASSURE((desc_flat.nr_ext % 6 == 0) &&
00681                        (desc_flat.ex_badpix == FALSE) &&
00682                        (desc_flat.fits_type == f2d_fits) &&
00683                        (desc_flat.frame_type == detector_frame),
00684                        CPL_ERROR_ILLEGAL_INPUT,
00685                        "MASTER_FLAT isn't in the correct format!!!");
00686 
00687         // load desc for XCAL and check
00688         KMO_TRY_EXIT_IF_NULL(
00689             xcalFrame = kmo_dfs_get_frame(frameset, XCAL));
00690         desc_xcal = kmo_identify_fits_header(cpl_frame_get_filename(xcalFrame));
00691         KMO_TRY_CHECK_ERROR_STATE_MSG("XCAL frame doesn't seem to "
00692                                       "be in KMOS-format!");
00693         KMO_TRY_ASSURE((desc_xcal.nr_ext % 3 == 0) &&
00694                        (desc_xcal.ex_badpix == FALSE) &&
00695                        (desc_xcal.fits_type == f2d_fits) &&
00696                        (desc_xcal.frame_type == detector_frame),
00697                        CPL_ERROR_ILLEGAL_INPUT,
00698                        "XCAL isn't in the correct format!!!");
00699         KMO_TRY_ASSURE((desc_xcal.naxis1 == nx) &&
00700                        (desc_xcal.naxis2 == ny),
00701                        CPL_ERROR_ILLEGAL_INPUT,
00702                        "MASTER_DARK and XCAL frame haven't same dimensions! "
00703                        "(x,y): (%d,%d) vs (%d,%d)",
00704                        nx, ny, desc_xcal.naxis1, desc_xcal.naxis2);
00705 
00706         nr_devices = desc_xcal.nr_ext;
00707 
00708         // load desc for YCAL and check
00709         KMO_TRY_EXIT_IF_NULL(
00710             ycalFrame = kmo_dfs_get_frame(frameset, YCAL));
00711         desc_ycal = kmo_identify_fits_header(cpl_frame_get_filename(ycalFrame));
00712         KMO_TRY_CHECK_ERROR_STATE_MSG("YCAL frame doesn't seem to "
00713                                       "be in KMOS-format!");
00714         KMO_TRY_ASSURE((desc_ycal.nr_ext % 3 == 0) &&
00715                        (desc_ycal.ex_badpix == FALSE) &&
00716                        (desc_ycal.fits_type == f2d_fits) &&
00717                        (desc_ycal.frame_type == detector_frame),
00718                        CPL_ERROR_ILLEGAL_INPUT,
00719                        "YCAL isn't in the correct format!!!");
00720         KMO_TRY_ASSURE((desc_ycal.naxis1 == nx) &&
00721                        (desc_ycal.naxis2 == ny) &&
00722                        (desc_ycal.nr_ext == nr_devices),
00723                        CPL_ERROR_ILLEGAL_INPUT,
00724                        "MASTER_DARK and YCAL frame haven't same dimensions! "
00725                        "(x,y): (%d,%d) vs (%d,%d)",
00726                        nx, ny, desc_ycal.naxis1, desc_ycal.naxis2);
00727 
00728         // load desc for LCAL and check
00729         KMO_TRY_EXIT_IF_NULL(
00730             lcalFrame = kmo_dfs_get_frame(frameset, LCAL));
00731         desc_lcal = kmo_identify_fits_header(cpl_frame_get_filename(lcalFrame));
00732         KMO_TRY_CHECK_ERROR_STATE_MSG("LCAL frame doesn't seem to "
00733                                       "be in KMOS-format!");
00734         KMO_TRY_ASSURE((desc_lcal.nr_ext % 3 == 0) &&
00735                        (desc_lcal.ex_badpix == FALSE) &&
00736                        (desc_lcal.fits_type == f2d_fits) &&
00737                        (desc_lcal.frame_type == detector_frame),
00738                        CPL_ERROR_ILLEGAL_INPUT,
00739                        "LCAL isn't in the correct format!!!");
00740         KMO_TRY_ASSURE((desc_lcal.naxis1 == nx) &&
00741                        (desc_lcal.naxis2 == ny) &&
00742                        (desc_lcal.nr_ext == nr_devices),
00743                        CPL_ERROR_ILLEGAL_INPUT,
00744                        "MASTER_DARK and LCAL frame haven't same dimensions! "
00745                        "(x,y): (%d,%d) vs (%d,%d)",
00746                        nx, ny, desc_lcal.naxis1, desc_lcal.naxis2);
00747         KMO_TRY_EXIT_IF_NULL(
00748             tmp_header = kmo_dfs_load_primary_header(frameset, LCAL));
00749 
00750         // load desc for FLAT_SKY and check
00751         nr_devices = KMOS_NR_DETECTORS;
00752         KMO_TRY_EXIT_IF_NULL(
00753             frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00754 
00755         KMO_TRY_EXIT_IF_NULL(
00756             main_header = kmclipm_propertylist_load(
00757                                          cpl_frame_get_filename(frame), 0));
00758         rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00759         KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00760         kmclipm_strip_angle(&rotangle);
00761         cpl_propertylist_delete(main_header); main_header = NULL;
00762 
00763         cnt = 1;
00764         while (frame != NULL) {
00765             KMO_TRY_EXIT_IF_NULL(
00766                 main_header = kmclipm_propertylist_load(
00767                                              cpl_frame_get_filename(frame), 0));
00768 
00769             desc_sky = kmo_identify_fits_header(
00770                         cpl_frame_get_filename(frame));
00771             KMO_TRY_CHECK_ERROR_STATE_MSG("FLAT_SKY frame doesn't seem to "
00772                                           "be in KMOS-format!");
00773             KMO_TRY_ASSURE((desc_sky.nr_ext == 3) &&
00774                            (desc_sky.ex_badpix == FALSE) &&
00775                            (desc_sky.fits_type == raw_fits) &&
00776                            (desc_sky.frame_type == detector_frame),
00777                            CPL_ERROR_ILLEGAL_INPUT,
00778                            "FLAT_SKY isn't in the correct format!!!");
00779             KMO_TRY_ASSURE((desc_sky.naxis1 == nx) &&
00780                            (desc_sky.naxis2 == ny) &&
00781                            (desc_sky.nr_ext == nr_devices),
00782                            CPL_ERROR_ILLEGAL_INPUT,
00783                            "MASTER_DARK and FLAT_SKY (no. %d) frame haven't "
00784                            "same dimensions! (x,y): (%d,%d) vs (%d,%d)",
00785                            cnt, nx, ny, desc_flat.naxis1, desc_flat.naxis2);
00786             kmo_free_fits_desc(&desc_sky);
00787             kmo_init_fits_desc(&desc_sky);
00788 
00789             KMO_TRY_ASSURE(
00790                 (kmo_check_lamp(main_header, INS_LAMP1_ST) == FALSE) &&
00791                 (kmo_check_lamp(main_header, INS_LAMP2_ST) == FALSE) &&
00792                 (kmo_check_lamp(main_header, INS_LAMP3_ST) == FALSE) &&
00793                 (kmo_check_lamp(main_header, INS_LAMP4_ST) == FALSE),
00794                 CPL_ERROR_ILLEGAL_INPUT,
00795                 "All lamps must be switched off for the FLAT_SKY frames!");
00796 
00797             // assert that filters have correct IDs and that all detectors of
00798             // all input frames have the same filter set
00799             for (int i = 1; i <= KMOS_NR_DETECTORS; i++) {
00800                 // ESO INS FILTi ID
00801                 KMO_TRY_EXIT_IF_NULL(
00802                     keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, i, IFU_FILTID_POSTFIX));
00803                 KMO_TRY_EXIT_IF_NULL(
00804                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00805 
00806                 KMO_TRY_EXIT_IF_NULL(
00807                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00808                 cpl_free(keyword); keyword = NULL;
00809 
00810                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00811                                (strcmp(filter_id, "YJ") == 0) ||
00812                                (strcmp(filter_id, "H") == 0) ||
00813                                (strcmp(filter_id, "K") == 0) ||
00814                                (strcmp(filter_id, "HK") == 0),
00815                                CPL_ERROR_ILLEGAL_INPUT,
00816                                "Filter ID in primary header must be either 'IZ', "
00817                                "'YJ', 'H', 'K' or " "'HK' !");
00818 
00819                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00820                                CPL_ERROR_ILLEGAL_INPUT,
00821                                "Filter IDs must be the same for FLAT_SKY frame"
00822                                " and lcal frame!"
00823                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00824                                i, cpl_frame_get_filename(frame),
00825                                filter_id, filter_id_l);
00826 
00827                 // ESO INS GRATi ID
00828                 KMO_TRY_EXIT_IF_NULL(
00829                     keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, i, IFU_GRATID_POSTFIX));
00830                 KMO_TRY_EXIT_IF_NULL(
00831                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00832 
00833                 KMO_TRY_EXIT_IF_NULL(
00834                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00835                 cpl_free(keyword); keyword = NULL;
00836 
00837                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00838                                (strcmp(filter_id, "YJ") == 0) ||
00839                                (strcmp(filter_id, "H") == 0) ||
00840                                (strcmp(filter_id, "K") == 0) ||
00841                                (strcmp(filter_id, "HK") == 0),
00842                                CPL_ERROR_ILLEGAL_INPUT,
00843                                "Grating ID in primary header must be either "
00844                                "'IZ', 'YJ', 'H', 'K' or " "'HK' !");
00845 
00846                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00847                                CPL_ERROR_ILLEGAL_INPUT,
00848                                "Grating IDs must be the same for FLAT_SKY frame"
00849                                " and lcal frame!"
00850                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00851                                i, cpl_frame_get_filename(frame),
00852                                filter_id, filter_id_l);
00853 
00854                 tmp_rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00855                 KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00856                 kmclipm_strip_angle(&tmp_rotangle);
00857                 KMO_TRY_ASSURE((abs(rotangle - tmp_rotangle) < 10.0) ||
00858                                (abs(rotangle - tmp_rotangle) > 360.-10.) ,
00859                         CPL_ERROR_ILLEGAL_INPUT,
00860                         "OCS ROT NAANGLE of sky flat frames differ to much: %f %f",
00861                         rotangle, tmp_rotangle);
00862             }
00863             cpl_propertylist_delete(main_header); main_header = NULL;
00864 
00865             // get next FLAT_SKY frame
00866             frame = kmo_dfs_get_frame(frameset, NULL);
00867             KMO_TRY_CHECK_ERROR_STATE();
00868             cnt++;
00869         }
00870 
00871         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00872 
00873         //
00874         // noise will be propagated when:
00875         // MASTER_DARK and MASTER_FLAT have noise extensions and if at least
00876         // 2 FLAT_SKY frames are provided.
00877         // Otherwise noise will be ignored.
00878         //
00879         if (desc_dark.ex_noise &&
00880             desc_flat.ex_noise &&
00881             (cpl_frameset_count_tags(frameset, FLAT_SKY) >= 2)) {
00882             process_noise = TRUE;
00883         }
00884 
00885         if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00886             cpl_msg_warning(cpl_func, "cmethod is changed to 'average' "
00887                             "since there is only one input frame! (The output "
00888                             "file won't have any noise extensions)");
00889 
00890             cmethod = "average";
00891         }
00892 
00893         //
00894         // Check whether 1st FLAT_SKY should be omitted
00895         //
00896         KMO_TRY_EXIT_IF_NULL(
00897             frameset_sky = cpl_frameset_new());
00898 
00899         if (add_all_sky) {
00900             // just add all FLAT_SKY frames without check
00901             frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00902             while (frame != NULL) {
00903                 KMO_TRY_EXIT_IF_ERROR(
00904                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00905                 frame = kmo_dfs_get_frame(frameset, NULL);
00906             }
00907             cpl_msg_info("", "Add all FLAT_SKY without checking for acquisition frame.");
00908         } else {
00909             // check if 1st FLAT_SKY has different exposure time and whether to omit it
00910             KMO_TRY_EXIT_IF_NULL(
00911                 frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00912 
00913             if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00914                 // just one FLAT_SKY, always add
00915                 KMO_TRY_EXIT_IF_ERROR(
00916                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00917                 KMO_TRY_CHECK_ERROR_STATE();
00918             } else {
00919                 // several FLAT_SKY frames, check exptime
00920 
00921                 // get exptime 1
00922                 KMO_TRY_EXIT_IF_NULL(
00923                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00924                 exptime1 = cpl_propertylist_get_double(main_header, EXPTIME);
00925                 KMO_TRY_CHECK_ERROR_STATE();
00926                 cpl_propertylist_delete(main_header); main_header = NULL;
00927 
00928                 // get exptime 2
00929                 frame = kmo_dfs_get_frame(frameset, NULL);
00930                 KMO_TRY_EXIT_IF_NULL(
00931                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00932                 exptime2 = cpl_propertylist_get_double(main_header, EXPTIME);
00933                 KMO_TRY_CHECK_ERROR_STATE();
00934                 cpl_propertylist_delete(main_header); main_header = NULL;
00935 
00936                 // loop remaining frames
00937                 same_exptime = TRUE;
00938                 frame = kmo_dfs_get_frame(frameset, NULL);
00939                 while (same_exptime && (frame != NULL)) {
00940                     KMO_TRY_EXIT_IF_NULL(
00941                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00942                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00943                     KMO_TRY_CHECK_ERROR_STATE();
00944                     cpl_propertylist_delete(main_header); main_header = NULL;
00945                     if (fabs(exptime-exptime2) > 0.01) {
00946                         // not same
00947                         same_exptime = FALSE;
00948                     }
00949                     frame = kmo_dfs_get_frame(frameset, NULL);
00950                 }
00951 
00952                 if (same_exptime) {
00953                     // frame [2,n] have same exptime, add them
00954                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00955                     KMO_TRY_EXIT_IF_NULL(
00956                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00957                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00958                     KMO_TRY_CHECK_ERROR_STATE();
00959                     cpl_propertylist_delete(main_header); main_header = NULL;
00960                     cpl_msg_info("", "Omit FLAT_SKY: %s with EXPTIME of %g sec (acquisition), other frame(s) have EXPTIME of %g sec", cpl_frame_get_filename(frame), exptime, exptime2);
00961                     frame = kmo_dfs_get_frame(frameset, NULL);
00962                     while (frame != NULL) {
00963                         KMO_TRY_EXIT_IF_ERROR(
00964                             cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00965                         frame = kmo_dfs_get_frame(frameset, NULL);
00966                     }
00967                     if (fabs(exptime1-exptime2) < 0.01) {
00968                         cpl_msg_warning("", "The 1st FLAT_SKY has the same exposure time as the following ones. "
00969                                             "It has anyway been omitted since we assume it is an acquisition frame. "
00970                                             "If you want to add it anyway call this recipe with the --add-all parameter");
00971                     }
00972                 } else {
00973                     cpl_msg_error("", "The exposure times of the FLAT_SKY frames don't match!");
00974                     cpl_msg_error("", "We assume that the 1st frame is an acquisition frame and would be omitted.");
00975                     cpl_msg_error("", "The following frames should have the same exposure time if they originate from the same template.");
00976                     cpl_msg_error("", "If you want to reduce them anyway call this recipe with the --add-all parameter");
00977                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00978                     while (frame != NULL) {
00979                         KMO_TRY_EXIT_IF_NULL(
00980                             main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00981                         exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00982                         KMO_TRY_CHECK_ERROR_STATE();
00983                         cpl_propertylist_delete(main_header); main_header = NULL;
00984                         cpl_msg_error("", "FLAT_SKY: %s, EXPTIME: %g", cpl_frame_get_filename(frame), exptime);
00985                         frame = kmo_dfs_get_frame(frameset, NULL);
00986                     }
00987                     cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_INPUT);
00988                     KMO_TRY_CHECK_ERROR_STATE();
00989                 }
00990             }
00991         }
00992 
00993         KMO_TRY_EXIT_IF_NULL(
00994             frame = kmo_dfs_get_frame(frameset_sky, FLAT_SKY));
00995         KMO_TRY_EXIT_IF_NULL(
00996             main_header = kmo_dfs_load_primary_header(frameset_sky, FLAT_SKY));
00997         KMO_TRY_EXIT_IF_NULL(
00998             keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, 1, IFU_GRATID_POSTFIX));
00999         KMO_TRY_EXIT_IF_NULL(
01000             filter = cpl_sprintf("%s", cpl_propertylist_get_string(main_header, keyword)));
01001         cpl_free(keyword); keyword = NULL;
01002 
01003         //
01004         // set default band-specific ranges for collapsing
01005         //
01006         if (ranges == NULL) {
01007             if (strcmp(filter, "IZ") == 0) {
01008                 ranges_txt = "0.81,1.05";
01009             } else if (strcmp(filter, "YJ") == 0) {
01010                 ranges_txt = "1.025,1.3";
01011             } else if (strcmp(filter, "H") == 0) {
01012                 ranges_txt = "1.5,1.7";
01013             } else if (strcmp(filter, "K") == 0) {
01014                 ranges_txt = "2.1,2.35";
01015             } else if (strcmp(filter, "HK") == 0) {
01016                 ranges_txt = "1.5,1.7;2.1,2.35";
01017 //                ranges_txt = "1.5,1.7";
01018             } else {
01019                 KMO_TRY_ASSURE(1 == 0,
01020                                CPL_ERROR_ILLEGAL_INPUT,
01021                                "We really shouldn't get here...");
01022             }
01023             cpl_msg_info("", "Spectral range to collapse has been set to %s um for this band.", ranges_txt);
01024             ranges = kmo_identify_ranges(ranges_txt);
01025             KMO_TRY_CHECK_ERROR_STATE();
01026         }
01027 
01028         // setup grid definition, wavelength start and end points will be set
01029         // in the detector loop
01030         KMO_TRY_EXIT_IF_ERROR(
01031             kmclipm_setup_grid(&gd, method, neighborhoodRange, pix_scale));
01032 
01033         // create filename for LUT
01034         KMO_TRY_EXIT_IF_NULL(
01035             fn_lut = cpl_sprintf("%s%s", "lut", suffix));
01036 
01037         // extract bounds
01038         KMO_TRY_EXIT_IF_NULL(
01039             tmp_header = kmclipm_propertylist_load(
01040                                          cpl_frame_get_filename(xcalFrame), 0));
01041         KMO_TRY_EXIT_IF_NULL(
01042             bounds = kmclipm_extract_bounds(tmp_header));
01043         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
01044 
01045         // get timestamps of xcal, ycal & lcal
01046         KMO_TRY_EXIT_IF_NULL(
01047             calTimestamp = kmo_get_timestamps(xcalFrame, ycalFrame, lcalFrame));
01048 
01049         // create arrays to hold reconstructed data and noise cubes and
01050         // their headers
01051         KMO_TRY_EXIT_IF_NULL(
01052             stored_data_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01053                                                             sizeof(cpl_imagelist*)));
01054         KMO_TRY_EXIT_IF_NULL(
01055             stored_noise_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01056                                                              sizeof(cpl_imagelist*)));
01057         KMO_TRY_EXIT_IF_NULL(
01058             stored_data_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01059                                                          sizeof(cpl_image*)));
01060         KMO_TRY_EXIT_IF_NULL(
01061             stored_noise_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01062                                                           sizeof(cpl_image*)));
01063         KMO_TRY_EXIT_IF_NULL(
01064             stored_sub_data_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01065                                                                      sizeof(cpl_propertylist*)));
01066         KMO_TRY_EXIT_IF_NULL(
01067             stored_sub_noise_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01068                                                                       sizeof(cpl_propertylist*)));
01069         KMO_TRY_EXIT_IF_NULL(
01070             edge_table_sky = (cpl_table***)cpl_calloc(KMOS_NR_DETECTORS,
01071                                                       sizeof(cpl_table**)));
01072         KMO_TRY_EXIT_IF_NULL(
01073             edge_table_flat = (cpl_table**)cpl_calloc(KMOS_IFUS_PER_DETECTOR,
01074                                                       sizeof(cpl_table*)));
01075         KMO_TRY_EXIT_IF_NULL(
01076             calAngles = cpl_vector_new(3));
01077 
01078         //
01079         // loop through all detectors
01080         //
01081         for (int det_nr = 1; det_nr <= nr_devices; det_nr++) {
01082             cpl_msg_info("","Processing detector No. %d", det_nr);
01083 
01084             KMO_TRY_EXIT_IF_NULL(
01085                 detector_in = cpl_imagelist_new());
01086 
01087             // load data of det_nr of all FLAT_SKY frames into an imagelist
01088             KMO_TRY_EXIT_IF_NULL(
01089                 img_in = kmo_dfs_load_image(frameset_sky, FLAT_SKY, det_nr, FALSE, TRUE, NULL));
01090 
01091             cnt = 0;
01092             while (img_in != NULL) {
01093                 cpl_imagelist_set(detector_in, img_in, cnt);
01094                 KMO_TRY_CHECK_ERROR_STATE();
01095 
01096                 /* load same extension of next FLAT_SKY frame*/
01097                 img_in = kmo_dfs_load_image(frameset_sky, NULL, det_nr, FALSE, TRUE, NULL);
01098                 KMO_TRY_CHECK_ERROR_STATE();
01099 
01100                 cnt++;
01101             }
01102 
01103             //
01104             // process imagelist
01105             //
01106 
01107             // combine imagelist (data only) and create noise (stdev of data)
01108             cpl_msg_info("","Combining frames...");
01109             if (process_noise) {
01110                 KMO_TRY_EXIT_IF_ERROR(
01111                     kmclipm_combine_frames(detector_in,
01112                                            NULL,
01113                                            NULL,
01114                                            cmethod,
01115                                            cpos_rej,
01116                                            cneg_rej,
01117                                            citer,
01118                                            cmax,
01119                                            cmin,
01120                                            &combined_data,
01121                                            &combined_noise,
01122                                            -1.0));
01123             } else {
01124                 KMO_TRY_EXIT_IF_ERROR(
01125                     kmclipm_combine_frames(detector_in,
01126                                            NULL,
01127                                            NULL,
01128                                            cmethod,
01129                                            cpos_rej,
01130                                            cneg_rej,
01131                                            citer,
01132                                            cmax,
01133                                            cmin,
01134                                            &combined_data,
01135                                            NULL,
01136                                            -1.0));
01137             }
01138 
01139             if (kmclipm_omit_warning_one_slice > 10) {
01140                 cpl_msg_warning(cpl_func, "Previous warning (number of "
01141                                           "identified slices) occured %d times.",
01142                                 kmclipm_omit_warning_one_slice);
01143                 kmclipm_omit_warning_one_slice = FALSE;
01144             }
01145 
01146             cpl_imagelist_delete(detector_in); detector_in = NULL;
01147 
01148             // load calibration files
01149             KMO_TRY_EXIT_IF_NULL(
01150                 xcal = kmo_dfs_load_cal_image(frameset, XCAL, det_nr, FALSE, rotangle,
01151                                               FALSE, NULL, &rotangle_found));
01152 
01153             KMO_TRY_EXIT_IF_ERROR(
01154                 cpl_vector_set(calAngles, 0, rotangle_found));
01155             KMO_TRY_EXIT_IF_NULL(
01156                 ycal = kmo_dfs_load_cal_image(frameset, YCAL, det_nr, FALSE, rotangle,
01157                                               FALSE, NULL, &rotangle_found));
01158             KMO_TRY_EXIT_IF_ERROR(
01159                 cpl_vector_set(calAngles, 1, rotangle_found));
01160             KMO_TRY_EXIT_IF_NULL(
01161                 lcal = kmo_dfs_load_cal_image(frameset, LCAL, det_nr, FALSE, rotangle,
01162                                               FALSE, NULL, &rotangle_found));
01163             KMO_TRY_EXIT_IF_ERROR(
01164                 cpl_vector_set(calAngles, 2, rotangle_found));
01165 
01166             // load bad pixel mask from XCAL and set NaNs to 0 and all other values to 1
01167             KMO_TRY_EXIT_IF_NULL(
01168                 bad_pix_mask = cpl_image_duplicate(xcal));
01169 
01170             KMO_TRY_EXIT_IF_NULL(
01171                 pbad_pix_mask = cpl_image_get_data_float(bad_pix_mask));
01172             for (int x = 0; x < nx; x++) {
01173                 for (int y = 0; y < ny; y++) {
01174                     if (isnan(pbad_pix_mask[x+nx*y])) {
01175                         pbad_pix_mask[x+nx*y] = 0.;
01176                     } else {
01177                         pbad_pix_mask[x+nx*y] = 1.;
01178                     }
01179                 }
01180             }
01181             KMO_TRY_CHECK_ERROR_STATE();
01182 
01183             //
01184             // calculate SKYFLAT_EDGE
01185             //
01186             if (has_flat_edge) {
01187                 // get edge-edgepars from FLAT_SKY
01188                 KMO_TRY_EXIT_IF_ERROR(
01189                     kmo_calc_edgepars(combined_data,
01190                                       unused_ifus_after[det_nr-1],
01191                                       bad_pix_mask,
01192                                       det_nr,
01193                                       &slitlet_ids,
01194                                       &edgepars));
01195                 KMO_TRY_CHECK_ERROR_STATE();
01196 
01197                 // copy edgepars to table for saving later on
01198                 KMO_TRY_EXIT_IF_NULL(
01199                     edge_table_sky[det_nr-1] = kmo_edgepars_to_table(slitlet_ids, edgepars));
01200 
01201                 if (edgepars != NULL) {
01202                     for (int i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01203                         cpl_matrix_delete(edgepars[i]); edgepars[i] = NULL;
01204                     }
01205                     cpl_free(edgepars); edgepars = NULL;
01206                 }
01207                 if (slitlet_ids != NULL) {
01208                     for (int i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01209                         cpl_vector_delete(slitlet_ids[i]); slitlet_ids[i] = NULL;
01210                     }
01211                     cpl_free(slitlet_ids); slitlet_ids = NULL;
01212                 }
01213                 KMO_TRY_CHECK_ERROR_STATE();
01214 
01215                 //
01216                 // correlate FLAT_EDGE and SKYFLAT_EDGE
01217                 //
01218 
01219                 // load flat_edge from MASTER_FLAT
01220                 KMO_TRY_EXIT_IF_NULL(
01221                     frame = kmo_dfs_get_frame(frameset, FLAT_EDGE));
01222                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01223                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01224                     KMO_TRY_EXIT_IF_NULL(
01225                         edge_table_flat[j] = kmclipm_cal_table_load(cpl_frame_get_filename(frame),
01226                                                                     ifu_nr, rotangle, 0, &tmp_rotangle));
01227                 }
01228 
01229                 //
01230                 // calculate shift value
01231                 //
01232 
01233                 KMO_TRY_EXIT_IF_NULL(
01234                     shift_vec = cpl_vector_new(KMOS_IFUS_PER_DETECTOR));
01235                 KMO_TRY_EXIT_IF_NULL(
01236                     edge_vec = cpl_vector_new(2*KMOS_SLITLET_X));
01237 
01238                 // get shift values for each IFU by comparing all edge parameters,
01239                 // rejecting and applying median
01240                 int row = 1024; // middle of frame
01241                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01242                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01243                     for (int edgeNr = 0; edgeNr < 2*KMOS_SLITLET_X; edgeNr++) {
01244                         double flatval = kmo_calc_fitted_slitlet_edge(edge_table_flat[j], edgeNr, row);
01245                         double skyval  = kmo_calc_fitted_slitlet_edge(edge_table_sky[det_nr-1][j], edgeNr, row);
01246                         cpl_vector_set(edge_vec, edgeNr, flatval-skyval);
01247                     }
01248 
01249                     // reject deviating edge-differences
01250                     kmclipm_vector *kv = NULL;
01251                     KMO_TRY_EXIT_IF_NULL(
01252                         kv = kmclipm_vector_create(cpl_vector_duplicate(edge_vec)));
01253                     kmclipm_reject_deviant(kv, 3, 3, NULL, NULL);
01254 
01255                     // set shift value for each IFU
01256                     cpl_vector_set(shift_vec, j, kmclipm_vector_get_median(kv, KMCLIPM_ARITHMETIC));
01257                     kmclipm_vector_delete(kv); kv = NULL;
01258                 }
01259                 cpl_vector_delete(edge_vec); edge_vec = NULL;
01260                 KMO_TRY_CHECK_ERROR_STATE();
01261 
01262                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01263                     cpl_table_delete(edge_table_flat[j]);
01264                     edge_table_flat[j] = NULL;
01265                 }
01266 
01267                 // take median of all IFU-shift-values
01268                 double shift_val = -cpl_vector_get_median(shift_vec);
01269                 cpl_vector_delete(shift_vec); shift_vec = NULL;
01270 
01271                 cpl_msg_info("", "Shift detector %d by %g pixels.", det_nr, shift_val);
01272 
01273                 int     xdim                = cpl_image_get_size_x(combined_data),
01274                         ydim                = cpl_image_get_size_y(combined_data);
01275                 double  *array_in           = cpl_calloc(xdim, sizeof(double)),
01276                         *array_out          = NULL;
01277                 float   *pcombined_data     = cpl_image_get_data_float(combined_data),
01278                         *pcombined_noise    = NULL;
01279     //            float   *tmpArray           = cpl_calloc(xdim, sizeof(float));
01280                 if (process_noise) {
01281                     pcombined_noise = cpl_image_get_data_float(combined_noise);
01282                 }
01283 
01284                 for (int iy = 0; iy < ydim; iy++) {
01285                     // cubic spline
01286                     for (int ix = 0; ix < xdim; ix++) {
01287                         array_in[ix] = pcombined_data[ix+iy*xdim];
01288                     }
01289                     array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01290                                                     xdim, shift_val, 1.0,
01291                                                     NATURAL);
01292                     for (int ix = 0; ix < xdim; ix++) {
01293                       pcombined_data[ix+iy*xdim] = array_out[ix];
01294                     }
01295                     cpl_free(array_out);
01296 
01297     //                // linear
01298     //                for (int ix = 1; ix < xdim; ix++) {
01299     //                    tmpArray[ix-1] = (pcombined_data[ix+iy*xdim]-pcombined_data[(ix-1)+iy*xdim])*shift_val +
01300     //                                     pcombined_data[(ix-1)+iy*xdim];
01301     //                }
01302     //                for (int ix = 1; ix < xdim; ix++) {
01303     //                    pcombined_data[ix+iy*xdim] = tmpArray[ix];
01304     //                }
01305 
01306                     if (process_noise) {
01307                         // cubic spline
01308                         for (int ix = 0; ix < xdim; ix++) {
01309                             array_in[ix] = pcombined_noise[ix+iy*xdim];
01310                         }
01311                         array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01312                                                         xdim, shift_val, 1.0,
01313                                                         NATURAL);
01314                         for (int ix = 0; ix < xdim; ix++) {
01315                           pcombined_noise[ix+iy*xdim] = array_out[ix];
01316                         }
01317                         cpl_free(array_out);
01318 
01319     //                    // linear
01320     //                    for (int ix = 1; ix < xdim; ix++) {
01321     //                        tmpArray[ix-1] = (pcombined_noise[ix+iy*xdim]-pcombined_noise[(ix-1)+iy*xdim])*shift_val +
01322     //                                         pcombined_noise[(ix-1)+iy*xdim];
01323     //                    }
01324     //                    for (int ix = 1; ix < xdim; ix++) {
01325     //                        pcombined_noise[ix+iy*xdim] = tmpArray[ix];
01326     //                    }
01327                     }
01328                 }
01329                 cpl_free(array_in); array_in = NULL;
01330             }
01331             //
01332             // reconstruct
01333             //
01334             // load MASTER_DARK and MASTER_FLAT
01335             KMO_TRY_EXIT_IF_NULL(
01336                 img_dark = kmo_dfs_load_image(frameset, MASTER_DARK,
01337                                               det_nr, FALSE, FALSE, NULL));
01338 
01339             if (process_noise) {
01340                 KMO_TRY_EXIT_IF_NULL(
01341                     img_dark_noise = kmo_dfs_load_image(frameset, MASTER_DARK,
01342                                                         det_nr, TRUE, FALSE, NULL));
01343             }
01344 
01345             KMO_TRY_EXIT_IF_NULL(
01346                 img_flat = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, FALSE,
01347                                                   rotangle, FALSE, NULL, &rotangle_found));
01348 
01349             if (process_noise) {
01350                 KMO_TRY_EXIT_IF_NULL(
01351                     img_flat_noise = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, TRUE,
01352                                                             rotangle, FALSE, NULL, &rotangle_found));
01353             }
01354 
01355             char *tmp_band_method = getenv("KMO_BAND_METHOD");
01356             int band_method = 0;
01357             if (tmp_band_method != NULL) {
01358                 band_method = atoi(tmp_band_method);
01359             }
01360 
01361             // ESO INS FILTi ID
01362             KMO_TRY_EXIT_IF_NULL(
01363                 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, det_nr,
01364                                       IFU_FILTID_POSTFIX));
01365             KMO_TRY_EXIT_IF_NULL(
01366                 filter_id = cpl_propertylist_get_string(main_header, keyword));
01367             cpl_free(keyword); keyword = NULL;
01368 
01369             KMO_TRY_EXIT_IF_NULL(
01370                 band_table = kmo_dfs_load_table(frameset, WAVE_BAND, 1, 0));
01371             KMO_TRY_EXIT_IF_ERROR(
01372                 kmclipm_setup_grid_band_lcal(&gd, lcal, filter_id, band_method,
01373                                              band_table));
01374             cpl_table_delete(band_table); band_table = NULL;
01375 
01376             cpl_msg_info("","Reconstructing cubes...");
01377             for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01378                 // update sub-header
01379                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01380 
01381                 // load raw image and sub-header
01382                 KMO_TRY_EXIT_IF_NULL(
01383                     sub_header = kmo_dfs_load_sub_header(frameset_sky, FLAT_SKY,
01384                                                          det_nr, FALSE));
01385 
01386                 KMO_TRY_EXIT_IF_NULL(
01387                     punused_ifus = cpl_array_get_data_int_const(
01388                                                   unused_ifus_after[det_nr-1]));
01389 
01390                 // check if IFU is valid according to main header keywords &
01391                 // calibration files
01392                 KMO_TRY_EXIT_IF_NULL(
01393                     keyword = cpl_sprintf("%s%d%s", IFU_VALID_PREFIX, ifu_nr,
01394                                           IFU_VALID_POSTFIX));
01395                 KMO_TRY_CHECK_ERROR_STATE();
01396                 ranges_txt = cpl_propertylist_get_string(main_header, keyword);
01397                 cpl_free(keyword); keyword = NULL;
01398 
01399                 if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) &&
01400                     (bounds[2*(ifu_nr-1)] != -1) &&
01401                     (bounds[2*(ifu_nr-1)+1] != -1) &&
01402                     (punused_ifus[j] == 0))
01403                 {
01404                     // IFU is valid
01405                     cpl_error_reset();
01406 
01407                     // calculate WCS
01408                     KMO_TRY_EXIT_IF_ERROR(
01409                         kmo_calc_wcs_gd(main_header, sub_header, ifu_nr, gd));
01410 
01411                     // reconstruct data
01412                     KMO_TRY_EXIT_IF_ERROR(
01413                         kmo_reconstruct_sci_image(ifu_nr,
01414                                                 bounds[2*(ifu_nr-1)],
01415                                                 bounds[2*(ifu_nr-1)+1],
01416                                                 combined_data,
01417                                                 combined_noise,
01418                                                 img_dark,
01419                                                 img_dark_noise,
01420                                                 img_flat,
01421                                                 img_flat_noise,
01422                                                 xcal,
01423                                                 ycal,
01424                                                 lcal,
01425                                                 &gd,
01426                                                 calTimestamp,
01427                                                 calAngles,
01428                                                 fn_lut,
01429                                                 &cube_data,
01430                                                 &cube_noise,
01431                                                 flux,
01432                                                 background));
01433                     KMO_TRY_CHECK_ERROR_STATE();
01434                 } else {
01435                     // IFU is invalid
01436                     cpl_error_reset();
01437                 } // if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) ...
01438 
01439                 // save output
01440                 KMO_TRY_EXIT_IF_NULL(
01441                     extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_DATA));
01442 
01443                 KMO_TRY_EXIT_IF_ERROR(
01444                     kmclipm_update_property_string(sub_header, EXTNAME,
01445                                                    extname,
01446                                                    "FITS extension name"));
01447 
01448                 cpl_free(extname); extname = NULL;
01449 
01450                 // store cube and sub header into array for later
01451                 stored_data_cubes[ifu_nr - 1] = cube_data;
01452                 stored_sub_data_headers[ifu_nr - 1] = sub_header;
01453 
01454                 if (process_noise) {
01455                     KMO_TRY_EXIT_IF_NULL(
01456                         sub_header = cpl_propertylist_duplicate(
01457                                            stored_sub_data_headers[ifu_nr - 1]));
01458                     KMO_TRY_EXIT_IF_NULL(
01459                         extname = kmo_extname_creator(ifu_frame, ifu_nr,
01460                                                       EXT_NOISE));
01461 
01462                     KMO_TRY_EXIT_IF_ERROR(
01463                         kmclipm_update_property_string(sub_header,
01464                                                 EXTNAME,
01465                                                 extname,
01466                                                 "FITS extension name"));
01467 
01468                     cpl_free(extname); extname = NULL;
01469 
01470                     stored_noise_cubes[ifu_nr - 1] = cube_noise;
01471                     stored_sub_noise_headers[ifu_nr - 1] = sub_header;
01472                 }
01473                 cpl_image_delete(data_ifu); data_ifu = NULL;
01474                 cpl_image_delete(noise_ifu); noise_ifu = NULL;
01475                 cube_data = NULL;
01476                 cube_noise = NULL;
01477             } // for j IFUs
01478 
01479             // free memory
01480             cpl_image_delete(combined_data); combined_data = NULL;
01481             cpl_image_delete(combined_noise); combined_noise = NULL;
01482             cpl_image_delete(xcal); xcal = NULL;
01483             cpl_image_delete(ycal); ycal = NULL;
01484             cpl_image_delete(lcal); lcal = NULL;
01485             cpl_image_delete(img_dark); img_dark = NULL;
01486             cpl_image_delete(img_flat); img_flat = NULL;
01487             cpl_image_delete(bad_pix_mask); bad_pix_mask = NULL;
01488             if (process_noise) {
01489                 cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01490                 cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01491             }
01492         } // for nr_devices
01493 
01494         cpl_free(edge_table_flat); edge_table_flat = NULL;
01495 
01496         // collapse cubes using rejection
01497         cpl_msg_info("","Collapsing cubes...");
01498         for (int det_nr = 1; det_nr <= nr_devices; det_nr++) {
01499             for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01500                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01501 
01502                 KMO_TRY_EXIT_IF_NULL(
01503                     punused_ifus = cpl_array_get_data_int_const(
01504                                                   unused_ifus_after[det_nr-1]));
01505                 if (punused_ifus[j] == 0) {
01506                     if (stored_sub_data_headers[ifu_nr-1] != NULL) {
01507                     // IFU is valid
01508                     ifu_crpix = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01509                                                             CRPIX3);
01510                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01511                                    "CRPIX3 keyword in FITS-header is missing!");
01512 
01513                     ifu_crval = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01514                                                             CRVAL3);
01515                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01516                                    "CRVAL3 keyword in FITS-header is missing!");
01517 
01518                     ifu_cdelt = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01519                                                             CDELT3);
01520                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01521                                    "CDELT3 keyword in FITS-header is missing!");
01522 
01523                     KMO_TRY_EXIT_IF_NULL(
01524                         identified_slices = kmo_identify_slices(ranges,
01525                                                                 ifu_crpix,
01526                                                                 ifu_crval,
01527                                                                 ifu_cdelt,
01528                                                                 gd.l.dim));
01529                     }/* else {
01530                         KMO_TRY_EXIT_IF_NULL(
01531                             identified_slices = cpl_vector_new(gd.l.dim));
01532                         cpl_vector_fill(identified_slices, 1.);
01533                     }*/
01534 
01535                     if (stored_data_cubes[ifu_nr-1] != NULL) {
01536                         KMO_TRY_EXIT_IF_ERROR(
01537                             kmclipm_make_image(stored_data_cubes[ifu_nr-1],
01538                                                stored_noise_cubes[ifu_nr-1],
01539                                                &stored_data_images[ifu_nr-1],
01540                                                &stored_noise_images[ifu_nr-1],
01541                                                identified_slices,
01542                                                cmethod, cpos_rej, cneg_rej,
01543                                                citer, cmax, cmin));
01544                     }
01545                     cpl_vector_delete(identified_slices); identified_slices = NULL;
01546                 } else {
01547                     // IFU is invalid
01548                 }
01549             }
01550         }
01551 
01552         // normalise all IFUs as a group.
01553         // Calculate mean of each IFU, add up and divide by number of successful
01554         // averaged IFUs.
01555         // Then divide all valid IFUs with mean value
01556         cnt = 0;
01557         for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01558             if (stored_data_images[i] != NULL) {
01559                 KMO_TRY_ASSURE(cpl_image_count_rejected(stored_data_images[i]) <
01560                                cpl_image_get_size_x(stored_data_images[i])*
01561                                cpl_image_get_size_y(stored_data_images[i]),
01562                                CPL_ERROR_ILLEGAL_INPUT,
01563                                "The collapsed, dark-subtracted image contains "
01564                                "only invalid values! Probably the provided "
01565                                "FLAT_SKY frames are exactly the same as the "
01566                                "frames used for MASTER_DARK calculation.");
01567 
01568                 mean_data += cpl_image_get_mean(stored_data_images[i]);
01569                 KMO_TRY_CHECK_ERROR_STATE();
01570                 cnt++;
01571             }
01572 
01573         }
01574         mean_data /= cnt;
01575 
01576         if (mean_data != 0.0) {
01577             for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01578                 if (stored_data_images[i] != NULL) {
01579                     KMO_TRY_EXIT_IF_ERROR(
01580                         cpl_image_divide_scalar(stored_data_images[i],
01581                                                 mean_data));
01582                 }
01583             }
01584         } else {
01585             cpl_msg_warning(cpl_func, "Data couldn't be normalised "
01586                                       "(mean = 0.0)!");
01587         }
01588 
01589         if (process_noise) {
01590             if (mean_data != 0.0) {
01591                 for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01592                     if (stored_noise_images[i] != NULL) {
01593                         KMO_TRY_EXIT_IF_ERROR(
01594                             cpl_image_divide_scalar(stored_noise_images[i],
01595                                                     mean_data));
01596                     }
01597                 }
01598             } else {
01599                 cpl_msg_warning(cpl_func, "Noise couldn't be normalised "
01600                                           "(mean = 0.0)!");
01601             }
01602         }
01603 
01604         // calculate qc parameters on normalised data
01605         qc_spat_unif = 0.0;
01606         cnt = 0;
01607         for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01608             if (stored_data_images[i] != NULL) {
01609                 tmp_mean = cpl_image_get_mean(stored_data_images[i]);
01610                 tmp_stdev = cpl_image_get_stdev (stored_data_images[i]);
01611 
01612                 qc_spat_unif += pow(tmp_mean-1, 2);
01613                 if (fabs(tmp_mean) > qc_max_dev) {
01614                     qc_max_dev = tmp_mean-1;
01615                     qc_max_dev_id = i+1;
01616                 }
01617                 if (fabs(tmp_stdev) > qc_max_nonunif) {
01618                     qc_max_nonunif = tmp_stdev;
01619                     qc_max_nonunif_id = i+1;
01620                 }
01621                 KMO_TRY_CHECK_ERROR_STATE();
01622                 cnt++;
01623             }
01624         }
01625         qc_spat_unif = sqrt(qc_spat_unif / cnt);
01626 
01627         //
01628         // save data
01629         //
01630 
01631         // update which IFUs are not used
01632         kmo_print_unused_ifus(unused_ifus_after, TRUE);
01633 
01634         KMO_TRY_EXIT_IF_ERROR(
01635             kmo_set_unused_ifus(unused_ifus_after, main_header,
01636                                 "kmo_illumination"));
01637 
01638         cpl_msg_info("","Saving data...");
01639 
01640         KMO_TRY_EXIT_IF_ERROR(
01641             kmclipm_update_property_double(main_header, QC_SPAT_UNIF, qc_spat_unif,
01642                                            "[adu] uniformity of illumination correction"));
01643         KMO_TRY_EXIT_IF_ERROR(
01644             kmclipm_update_property_double(main_header, QC_SPAT_MAX_DEV, qc_max_dev,
01645                                            "[adu] max. deviation from unity"));
01646         KMO_TRY_EXIT_IF_ERROR(
01647             kmclipm_update_property_int(main_header, QC_SPAT_MAX_DEV_ID, qc_max_dev_id,
01648                                         "[] IFU ID with max. dev. from unity"));
01649         KMO_TRY_EXIT_IF_ERROR(
01650             kmclipm_update_property_double(main_header, QC_SPAT_MAX_NONUNIF, qc_max_nonunif,
01651                                            "[adu] max. stdev of illumination corr."));
01652         KMO_TRY_EXIT_IF_ERROR(
01653             kmclipm_update_property_int(main_header, QC_SPAT_MAX_NONUNIF_ID, qc_max_nonunif_id,
01654                                         "[] IFU ID with max. stdev in illum. corr."));
01655 
01656         KMO_TRY_EXIT_IF_ERROR(
01657             kmo_dfs_save_main_header(frameset, ILLUM_CORR, suffix, frame,
01658                                      main_header, parlist, cpl_func));
01659 
01660         KMO_TRY_EXIT_IF_ERROR(
01661             kmo_dfs_save_main_header(frameset, SKYFLAT_EDGE, suffix, frame,
01662                                      main_header, parlist, cpl_func));
01663 
01664         for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01665             KMO_TRY_EXIT_IF_ERROR(
01666                 kmo_dfs_save_image(stored_data_images[i], ILLUM_CORR, suffix,
01667                                    stored_sub_data_headers[i], 0./0.));
01668 
01669             if (process_noise) {
01670                 KMO_TRY_EXIT_IF_ERROR(
01671                     kmo_dfs_save_image(stored_noise_images[i], ILLUM_CORR,
01672                                        suffix, stored_sub_noise_headers[i], 0./0.));
01673             }
01674         }
01675 
01676         for (int det_nr = 1; det_nr <= nr_devices; det_nr++) {
01677             for (int ifu_nr = 0; ifu_nr < KMOS_IFUS_PER_DETECTOR; ifu_nr++) {
01678                 KMO_TRY_EXIT_IF_ERROR(
01679                     kmclipm_update_property_int(stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01680                                                 CAL_IFU_NR,
01681                                                 ifu_nr+1+(det_nr-1)*KMOS_IFUS_PER_DETECTOR,
01682                                                 "IFU Number {1..24}"));
01683                 KMO_TRY_EXIT_IF_ERROR(
01684                     kmclipm_update_property_double(
01685                                                 stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01686                                                 CAL_ROTANGLE,
01687                                                 rotangle_found,
01688                                                 "[deg] Rotator relative to nasmyth"));
01689                 if (has_flat_edge) {
01690                     // save edge-parameters as product
01691                     KMO_TRY_EXIT_IF_ERROR(
01692                         kmo_dfs_save_table(edge_table_sky[det_nr-1][ifu_nr], SKYFLAT_EDGE, suffix,
01693                                            stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr]));
01694                 }
01695             }
01696         }
01697     }
01698     KMO_CATCH
01699     {
01700         KMO_CATCH_MSG();
01701         ret_val = -1;
01702     }
01703     kmo_free_fits_desc(&desc_sky);
01704     kmo_free_fits_desc(&desc_dark);
01705     kmo_free_fits_desc(&desc_flat);
01706     kmo_free_fits_desc(&desc_xcal);
01707     kmo_free_fits_desc(&desc_ycal);
01708     kmo_free_fits_desc(&desc_lcal);
01709     cpl_image_delete(combined_data); combined_data = NULL;
01710     cpl_image_delete(combined_noise); combined_noise = NULL;
01711     cpl_image_delete(xcal); xcal = NULL;
01712     cpl_image_delete(ycal); ycal = NULL;
01713     cpl_image_delete(lcal); lcal = NULL;
01714     cpl_image_delete(img_dark); img_dark = NULL;
01715     cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01716     cpl_image_delete(img_flat); img_flat = NULL;
01717     cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01718     cpl_array_delete(calTimestamp); calTimestamp = NULL;
01719     cpl_free(bounds); bounds = NULL;
01720     kmo_free_unused_ifus(unused_ifus_before); unused_ifus_before = NULL;
01721     kmo_free_unused_ifus(unused_ifus_after); unused_ifus_after = NULL;
01722     cpl_free(fn_lut); fn_lut = NULL;
01723     cpl_free(suffix); suffix = NULL;
01724     cpl_frameset_delete(frameset_sky); frameset_sky = NULL;
01725     cpl_vector_delete(ranges); ranges = NULL;
01726     cpl_free(filter); filter = NULL;
01727     if (calAngles != NULL) {
01728         cpl_vector_delete(calAngles); calAngles = NULL;
01729     }
01730     cpl_propertylist_delete(main_header); main_header = NULL;
01731     for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01732         if (stored_data_cubes != NULL) {
01733             cpl_imagelist_delete(stored_data_cubes[i]);
01734             stored_data_cubes[i] = NULL;
01735         }
01736         if (stored_noise_cubes != NULL) {
01737             cpl_imagelist_delete(stored_noise_cubes[i]);
01738             stored_noise_cubes[i] = NULL;
01739         }
01740         if (stored_data_images != NULL) {
01741             cpl_image_delete(stored_data_images[i]);
01742             stored_data_images[i] = NULL;
01743         }
01744         if (stored_noise_images != NULL) {
01745             cpl_image_delete(stored_noise_images[i]);
01746             stored_noise_images[i] = NULL;
01747         }
01748         if (stored_sub_data_headers != NULL) {
01749             cpl_propertylist_delete(stored_sub_data_headers[i]);
01750             stored_sub_data_headers[i] = NULL;
01751         }
01752         if (stored_sub_noise_headers != NULL) {
01753             cpl_propertylist_delete(stored_sub_noise_headers[i]);
01754             stored_sub_noise_headers[i] = NULL;
01755         }
01756     }
01757     cpl_free(stored_data_cubes); stored_data_cubes = NULL;
01758     cpl_free(stored_noise_cubes); stored_noise_cubes = NULL;
01759     cpl_free(stored_data_images); stored_data_images = NULL;
01760     cpl_free(stored_noise_images); stored_noise_images = NULL;
01761     cpl_free(stored_sub_data_headers); stored_sub_data_headers = NULL;
01762     cpl_free(stored_sub_noise_headers); stored_sub_noise_headers = NULL;
01763     if (edge_table_sky != NULL) {
01764         for (int i = 0; i < KMOS_NR_DETECTORS; i++) {
01765             if (edge_table_sky[i] != NULL) {
01766                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01767                     cpl_table_delete(edge_table_sky[i][j]);
01768                     edge_table_sky[i][j] = NULL;
01769                 }
01770                 cpl_free(edge_table_sky[i]); edge_table_sky[i] = NULL;
01771             }
01772         }
01773         cpl_free(edge_table_sky); edge_table_sky = NULL;
01774     }
01775     if (edge_table_flat != NULL) {
01776         for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01777             cpl_table_delete(edge_table_flat[j]);
01778             edge_table_flat[j] = NULL;
01779         }
01780         cpl_free(edge_table_flat); edge_table_flat = NULL;
01781     }
01782 
01783     return ret_val;
01784 }
01785