KMOS Pipeline Reference Manual  1.2.7
kmo_illumination.c
00001 /* $Id: kmo_illumination.c,v 1.65 2013-10-21 13:44:54 aagudo Exp $
00002  *
00003  * This file is part of the KMOS Pipeline
00004  * Copyright (C) 2002,2003 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: aagudo $
00023  * $Date: 2013-10-21 13:44:54 $
00024  * $Revision: 1.65 $
00025  * $Name: not supported by cvs2svn $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 /*-----------------------------------------------------------------------------
00033  *                              Includes
00034  *----------------------------------------------------------------------------*/
00035 
00036 #include <math.h>
00037 #include <string.h>
00038 
00039 #include <cpl.h>
00040 
00041 #include "kmo_priv_reconstruct.h"
00042 #include "kmo_priv_functions.h"
00043 #include "kmo_priv_flat.h"
00044 #include "kmo_priv_wave_cal.h"
00045 #include "kmclipm_priv_splines.h"
00046 #include "kmo_functions.h"
00047 #include "kmo_cpl_extensions.h"
00048 #include "kmo_dfs.h"
00049 #include "kmo_error.h"
00050 #include "kmo_constants.h"
00051 #include "kmo_debug.h"
00052 
00053 /*-----------------------------------------------------------------------------
00054  *                          Functions prototypes
00055  *----------------------------------------------------------------------------*/
00056 
00057 static int kmo_illumination_create(cpl_plugin *);
00058 static int kmo_illumination_exec(cpl_plugin *);
00059 static int kmo_illumination_destroy(cpl_plugin *);
00060 static int kmo_illumination(cpl_parameterlist *, cpl_frameset *);
00061 
00062 /*-----------------------------------------------------------------------------
00063  *                          Static variables
00064  *----------------------------------------------------------------------------*/
00065 
00066 static char kmo_illumination_description[] =
00067 "This recipe creates the spatial non-uniformity calibration frame needed for\n"
00068 "all three detectors. It must be called after the kmo_wave_cal-recipe, which\n"
00069 "generates the spectral calibration frame needed in this recipe. As input at\n"
00070 "least a sky, a master dark, a master flat and the spatial and spectral cali-\n"
00071 "bration frames are required.\n"
00072 "The created product, the illumination correction, can be used as input for\n"
00073 "kmo_std_star and kmo_sci_red.\n"
00074 "\n"
00075 "BASIC PARAMETERS:\n"
00076 "-----------------\n"
00077 "--imethod\n"
00078 "The interpolation method used for reconstruction.\n"
00079 "\n"
00080 "--range\n"
00081 "The spectral ranges to combine when collapsing the reconstructed cubes. e.g.\n"
00082 "\"x1_start,x1_end;x2_start,x2_end\" (microns)\n"
00083 "\n"
00084 "ADVANCED PARAMETERS\n"
00085 "-------------------\n"
00086 "--flux\n"
00087 "Specify if flux conservation should be applied.\n"
00088 "\n"
00089 "--add-all\n"
00090 "By default the first FLAT_SKY frame is omitted, since in the\n"
00091 "KMOS_spec_cal_skyflat template this is an acquisition frame to estimate the\n"
00092 "needed exposure time for the subsequent FLAT_SKY frames. If anyway all frames\n"
00093 "should be considered, set this parameter to TRUE.\n"
00094 "\n"
00095 "--neighborhoodRange\n"
00096 "Defines the range to search for neighbors during reconstruction\n"
00097 "\n"
00098 "--b_samples\n"
00099 "The number of samples in spectral direction for the reconstructed cube.\n"
00100 "Ideally this number should be greater than 2048, the detector size.\n"
00101 "\n"
00102 "--b_start\n"
00103 "--b_end\n"
00104 "Used to define manually the start and end wavelength for the reconstructed\n"
00105 "cube. By default the internally defined values are used.\n"
00106 "\n"
00107 "--cmethod\n"
00108 "Following methods of frame combination are available:\n"
00109 "   * 'ksigma' (Default)\n"
00110 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00111 "   are examined. If they deviate significantly, they will be rejected according\n"
00112 "   to the conditions:\n"
00113 "       val > mean + stdev * cpos_rej\n"
00114 "   and\n"
00115 "       val < mean - stdev * cneg_rej\n"
00116 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00117 "   parameters. In the first iteration median and percentile level are used.\n"
00118 "\n"
00119 "   * 'median'\n"
00120 "   At each pixel position the median is calculated.\n"
00121 "\n"
00122 "   * 'average'\n"
00123 "   At each pixel position the average is calculated.\n"
00124 "\n"
00125 "   * 'sum'\n"
00126 "   At each pixel position the sum is calculated.\n"
00127 "\n"
00128 "   * 'min_max'\n"
00129 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00130 "   --cmax and --cmin apply to this method.\n"
00131 "\n"
00132 "--cpos_rej\n"
00133 "--cneg_rej\n"
00134 "--citer\n"
00135 "see --cmethod='ksigma'\n"
00136 "\n"
00137 "--cmax\n"
00138 "--cmin\n"
00139 "see --cmethod='min_max'\n"
00140 "\n"
00141 "--pix_scale\n"
00142 "Change the pixel scale [arcsec]. Default of 0.2\" results into cubes of\n"
00143 "14x14pix, a scale of 0.1\" results into cubes of 28x28pix, etc.\n"
00144 "\n"
00145 "--suppress_extension\n"
00146 "If set to TRUE, the arbitrary filename extensions are supressed. If multiple\n"
00147 "products with the same category are produced, they will be numered consecutively\n"
00148 "starting from 0.\n"
00149 "\n"
00150 "-------------------------------------------------------------------------------\n"
00151 "  Input files:\n"
00152 "\n"
00153 "   DO                    KMOS                                                  \n"
00154 "   category              Type   Explanation                    Required #Frames\n"
00155 "   --------              -----  -----------                    -------- -------\n"
00156 "   FLAT_SKY               F2D   Sky exposures                     Y      1-n   \n"
00157 "                                (at least 3 frames recommended)                \n"
00158 "   MASTER_DARK            F2D   Master dark                       Y       1    \n"
00159 "   MASTER_FLAT            F2D   Master flat                       Y       1    \n"
00160 "   XCAL                   F2D   x calibration frame               Y       1    \n"
00161 "   YCAL                   F2D   y calibration frame               Y       1    \n"
00162 "   LCAL                   F2D   Wavelength calib. frame           Y       1    \n"
00163 "   WAVE_BAND              F2L   Table with start-/end-wavelengths Y       1    \n"
00164 "   FLAT_EDGE              F2L   Table with fitted slitlet edges   N      0,1   \n"
00165 "\n"
00166 "  Output files:\n"
00167 "\n"
00168 "   DO                    KMOS\n"
00169 "   category              Type   Explanation\n"
00170 "   --------              -----  -----------\n"
00171 "   ILLUM_CORR            F2I    Illumination calibration frame   \n"
00172 "   If FLAT_EDGE is provided: \n"
00173 "   SKYFLAT_EDGE          F2L    Frame containing parameters of fitted \n"
00174 "                                slitlets of all IFUs of all detectors\n"
00175 "-------------------------------------------------------------------------------\n"
00176 "\n";
00177 
00178 /*-----------------------------------------------------------------------------
00179  *                              Functions code
00180  *----------------------------------------------------------------------------*/
00181 
00198 int cpl_plugin_get_info(cpl_pluginlist *list)
00199 {
00200     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00201     cpl_plugin *plugin = &recipe->interface;
00202 
00203     cpl_plugin_init(plugin,
00204                         CPL_PLUGIN_API,
00205                         KMOS_BINARY_VERSION,
00206                         CPL_PLUGIN_TYPE_RECIPE,
00207                         "kmo_illumination",
00208                         "Create a calibration frame to correct spatial "
00209                         "non-uniformity of flatfield.",
00210                         kmo_illumination_description,
00211                         "Alex Agudo Berbel",
00212                         "kmos-spark@mpe.mpg.de",
00213                         kmos_get_license(),
00214                         kmo_illumination_create,
00215                         kmo_illumination_exec,
00216                         kmo_illumination_destroy);
00217 
00218     cpl_pluginlist_append(list, plugin);
00219 
00220     return 0;
00221 }
00222 
00230 static int kmo_illumination_create(cpl_plugin *plugin)
00231 {
00232     cpl_recipe *recipe;
00233     cpl_parameter *p;
00234 
00235     /* Check that the plugin is part of a valid recipe */
00236     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00237         recipe = (cpl_recipe *)plugin;
00238     else
00239         return -1;
00240 
00241     /* Create the parameters list in the cpl_recipe object */
00242     recipe->parameters = cpl_parameterlist_new();
00243 
00244     /* Fill the parameters list */
00245     /* --imethod */
00246     p = cpl_parameter_new_value("kmos.kmo_illumination.imethod",
00247                                 CPL_TYPE_STRING,
00248                                 "Method to use for interpolation: "
00249                                 "[\"NN\" (nearest neighbour), "
00250                                 "\"lwNN\" (linear weighted nearest neighbor), "
00251                                 "\"swNN\" (square weighted nearest neighbor), "
00252                                 "\"MS\" (Modified Shepard's method), "
00253                                 "\"CS\" (Cubic spline)]",
00254                                 "kmos.kmo_illumination",
00255                                 "CS");
00256     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "imethod");
00257     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00258     cpl_parameterlist_append(recipe->parameters, p);
00259 
00260     /* --neighborhoodRange */
00261     p = cpl_parameter_new_value("kmos.kmo_illumination.neighborhoodRange",
00262                                 CPL_TYPE_DOUBLE,
00263                                 "Defines the range to search for neighbors. "
00264                                 "in pixels",
00265                                 "kmos.kmo_illumination",
00266                                 1.001);
00267     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "neighborhoodRange");
00268     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00269     cpl_parameterlist_append(recipe->parameters, p);
00270 
00271     /* --range */
00272     p = cpl_parameter_new_value("kmos.kmo_illumination.range",
00273                                 CPL_TYPE_STRING,
00274                                 "The spectral ranges to combine when collapsing "
00275                                 "the reconstructed cubes. e.g. "
00276                                 "\"x1_start,x1_end;x2_start,x2_end\" (microns)",
00277                                 "kmos.kmo_illumination",
00278                                 "");
00279     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "range");
00280     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00281     cpl_parameterlist_append(recipe->parameters, p);
00282 
00283     /* --flux */
00284     p = cpl_parameter_new_value("kmos.kmo_illumination.flux",
00285                                 CPL_TYPE_BOOL,
00286                                 "TRUE: Apply flux conservation. FALSE: otherwise",
00287                                 "kmos.kmo_illumination",
00288                                 FALSE);
00289     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00290     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00291     cpl_parameterlist_append(recipe->parameters, p);
00292 
00293     /* --add-all */
00294     p = cpl_parameter_new_value("kmos.kmo_illumination.add-all",
00295                                 CPL_TYPE_BOOL,
00296                                 "FALSE: omit 1st FLAT_SKY frame (acquisition), "
00297                                 "TRUE: don't perform any checks, add them all",
00298                                 "kmos.kmo_illumination",
00299                                 FALSE);
00300     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "add-all");
00301     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00302     cpl_parameterlist_append(recipe->parameters, p);
00303 
00304     /* --pix_scale */
00305     p = cpl_parameter_new_value("kmos.kmo_illumination.pix_scale",
00306                                 CPL_TYPE_DOUBLE,
00307                                 "Change the pixel scale [arcsec]. "
00308                                 "Default of 0.2\" results into cubes of 14x14pix, "
00309                                 "a scale of 0.1\" results into cubes of 28x28pix, "
00310                                 "etc.",
00311                                 "kmos.kmo_illumination",
00312                                 KMOS_PIX_RESOLUTION);
00313     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pix_scale");
00314     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00315     cpl_parameterlist_append(recipe->parameters, p);
00316 
00317     /* --suppress_extension */
00318     p = cpl_parameter_new_value("kmos.kmo_illumination.suppress_extension",
00319                                 CPL_TYPE_BOOL,
00320                                 "Suppress arbitrary filename extension. "
00321                                 "(TRUE (apply) or FALSE (don't apply)",
00322                                 "kmos.kmo_illumination",
00323                                 FALSE);
00324     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "suppress_extension");
00325     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00326     cpl_parameterlist_append(recipe->parameters, p);
00327 
00328     // add parameters for band-definition
00329     kmo_band_pars_create(recipe->parameters,
00330                          "kmos.kmo_illumination");
00331 
00332     // add parameters for combining
00333     return kmo_combine_pars_create(recipe->parameters,
00334                                    "kmos.kmo_illumination",
00335                                    DEF_REJ_METHOD,
00336                                    FALSE);
00337 }
00338 
00344 static int kmo_illumination_exec(cpl_plugin *plugin)
00345 {
00346     cpl_recipe  *recipe;
00347 
00348     /* Get the recipe out of the plugin */
00349     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00350         recipe = (cpl_recipe *)plugin;
00351     else return -1;
00352 
00353     return kmo_illumination(recipe->parameters, recipe->frames);
00354 }
00355 
00361 static int kmo_illumination_destroy(cpl_plugin *plugin)
00362 {
00363     cpl_recipe *recipe;
00364 
00365     /* Get the recipe out of the plugin */
00366     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00367         recipe = (cpl_recipe *)plugin;
00368     else return -1 ;
00369 
00370     cpl_parameterlist_delete(recipe->parameters);
00371     return 0 ;
00372 }
00373 
00388 static int kmo_illumination(cpl_parameterlist *parlist, cpl_frameset *frameset)
00389 {
00390     int              ret_val                    = 0,
00391                      nr_devices                 = 0,
00392                      ifu_nr                     = 0,
00393                      nx                         = 0,
00394                      ny                         = 0,
00395                      process_noise              = FALSE,
00396                      cmax                       = 0,
00397                      cmin                       = 0,
00398                      citer                      = 0,
00399                      *bounds                    = NULL,
00400                      cnt                        = 0,
00401                      qc_max_dev_id              = 0,
00402                      qc_max_nonunif_id          = 0,
00403                      flux                       = FALSE,
00404                      background                 = FALSE,
00405                      add_all_sky                = FALSE,
00406                      same_exptime               = TRUE,
00407                      suppress_extension         = FALSE,
00408                      has_flat_edge              = FALSE,
00409                      i = 0, j = 0, x = 0, y = 0, ix = 0, iy = 0, det_nr = 0, edgeNr = 0;
00410     const int        *punused_ifus              = NULL;
00411     float            *pbad_pix_mask             = NULL;
00412     double           exptime                    = 0.,
00413                      exptime1                   = 0.,
00414                      exptime2                   = 0.,
00415                      cpos_rej                   = 0.0,
00416                      cneg_rej                   = 0.0,
00417                      neighborhoodRange          = 1.001,
00418                      mean_data                  = 0.0,
00419                      ifu_crpix                  = 0.0,
00420                      ifu_crval                  = 0.0,
00421                      ifu_cdelt                  = 0.0,
00422                      qc_spat_unif               = 0.0,
00423                      qc_max_dev                 = 0.0,
00424                      qc_max_nonunif             = 0.0,
00425                      tmp_stdev                  = 0.0,
00426                      tmp_mean                   = 0.0,
00427                      rotangle                   = 0.0,
00428                      tmp_rotangle               = 0.0,
00429                      rotangle_found             = 0.0,
00430                      pix_scale                  = 0.0;
00431     char             *keyword                   = NULL,
00432                      *fn_lut                    = NULL,
00433                      *suffix                    = NULL,
00434                      *fn_suffix                 = NULL,
00435                      *extname                   = NULL,
00436                      *filter                    = NULL;
00437     const char       *method                    = NULL,
00438                      *cmethod                   = NULL,
00439                      *filter_id_l               = NULL,
00440                      *filter_id                 = NULL,
00441                      *ranges_txt                = NULL;
00442     cpl_array        *calTimestamp              = NULL,
00443                      **unused_ifus_before       = NULL,
00444                      **unused_ifus_after        = NULL;
00445     cpl_frame        *frame                     = NULL,
00446                      *xcalFrame                 = NULL,
00447                      *ycalFrame                 = NULL,
00448                      *lcalFrame                 = NULL;
00449     cpl_frameset     *frameset_sky              = NULL;
00450     cpl_image        *img_in                    = NULL,
00451                      *img_dark                  = NULL,
00452                      *img_dark_noise            = NULL,
00453                      *img_flat                  = NULL,
00454                      *img_flat_noise            = NULL,
00455                      *combined_data             = NULL,
00456                      *combined_noise            = NULL,
00457                      *xcal                      = NULL,
00458                      *ycal                      = NULL,
00459                      *lcal                      = NULL,
00460                      *bad_pix_mask              = NULL,
00461                      *data_ifu                  = NULL,
00462                      *noise_ifu                 = NULL,
00463                      **stored_data_images       = NULL,
00464                      **stored_noise_images      = NULL;
00465     cpl_imagelist    *cube_data                 = NULL,
00466                      *cube_noise                = NULL,
00467                      *detector_in               = NULL,
00468                      **stored_data_cubes        = NULL,
00469                      **stored_noise_cubes       = NULL;
00470     cpl_matrix       **edgepars                 = NULL;
00471     cpl_propertylist *main_header               = NULL,
00472                      *tmp_header                = NULL,
00473                      *sub_header                = NULL,
00474                      **stored_sub_data_headers  = NULL,
00475                      **stored_sub_noise_headers = NULL;
00476     cpl_table        *band_table                = NULL,
00477                      ***edge_table_sky          = NULL,
00478                      **edge_table_flat          = NULL;
00479     cpl_vector       *ranges                    = NULL,
00480                      *identified_slices         = NULL,
00481                      *calAngles                 = NULL,
00482                      **slitlet_ids              = NULL,
00483                      *shift_vec                 = NULL,
00484                      *edge_vec                  = NULL;
00485     main_fits_desc   desc_sky,
00486                      desc_dark,
00487                      desc_flat,
00488                      desc_xcal,
00489                      desc_ycal,
00490                      desc_lcal;
00491     gridDefinition   gd;
00492 
00493     KMO_TRY
00494     {
00495         kmo_init_fits_desc(&desc_sky);
00496         kmo_init_fits_desc(&desc_dark);
00497         kmo_init_fits_desc(&desc_flat);
00498         kmo_init_fits_desc(&desc_xcal);
00499         kmo_init_fits_desc(&desc_ycal);
00500         kmo_init_fits_desc(&desc_lcal);
00501 
00502         /* --- check input --- */
00503         KMO_TRY_ASSURE((parlist != NULL) &&
00504                        (frameset != NULL),
00505                        CPL_ERROR_NULL_INPUT,
00506                        "Not all input data is provided!");
00507 
00508         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, FLAT_SKY) >= 1,
00509                        CPL_ERROR_ILLEGAL_INPUT,
00510                        "One or more FLAT_SKY frames are required!");
00511 
00512         if (cpl_frameset_count_tags(frameset, FLAT_SKY) < 3) {
00513             cpl_msg_warning(cpl_func, "It is recommended to provide at least "
00514                                       "3 FLAT_SKY frames!");
00515         }
00516 
00517         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_DARK) == 1,
00518                        CPL_ERROR_ILLEGAL_INPUT,
00519                        "Exactly one MASTER_DARK frame is required!");
00520 
00521         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_FLAT) == 1,
00522                        CPL_ERROR_ILLEGAL_INPUT,
00523                        "Exactly one MASTER_FLAT frame is required!");
00524 
00525         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, XCAL) == 1,
00526                        CPL_ERROR_ILLEGAL_INPUT,
00527                        "Exactly one XCAL frame is required!");
00528 
00529         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, YCAL) == 1,
00530                        CPL_ERROR_ILLEGAL_INPUT,
00531                        "Exactly one YCAL frame is required!");
00532 
00533         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, LCAL) == 1,
00534                        CPL_ERROR_ILLEGAL_INPUT,
00535                        "Exactly one LCAL frame is required!");
00536 
00537         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, WAVE_BAND) == 1,
00538                        CPL_ERROR_ILLEGAL_INPUT,
00539                        "Exactly one WAVE_BAND frame is required!");
00540 
00541         KMO_TRY_ASSURE((cpl_frameset_count_tags(frameset, FLAT_EDGE) == 1) ||
00542                        (cpl_frameset_count_tags(frameset, FLAT_EDGE) == 0),
00543                        CPL_ERROR_ILLEGAL_INPUT,
00544                        "Exactly one FLAT_EDGE frame is required!");
00545 
00546         has_flat_edge = cpl_frameset_count_tags(frameset, FLAT_EDGE);
00547 
00548         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_illumination") == 1,
00549                        CPL_ERROR_ILLEGAL_INPUT,
00550                        "Cannot identify RAW and CALIB frames!");
00551 
00552         /* --- get parameters --- */
00553         cpl_msg_info("", "--- Parameter setup for kmo_illumination ---");
00554 
00555         KMO_TRY_EXIT_IF_NULL(
00556             method = kmo_dfs_get_parameter_string(parlist,
00557                                               "kmos.kmo_illumination.imethod"));
00558 
00559         KMO_TRY_ASSURE((strcmp(method, "NN") == 0) ||
00560                        (strcmp(method, "lwNN") == 0) ||
00561                        (strcmp(method, "swNN") == 0) ||
00562                        (strcmp(method, "MS") == 0) ||
00563                        (strcmp(method, "CS") == 0),
00564                        CPL_ERROR_ILLEGAL_INPUT,
00565                        "method must be either \"NN\", \"lwNN\", "
00566                        "\"swNN\", \"MS\" or \"CS\"!");
00567 
00568         KMO_TRY_EXIT_IF_ERROR(
00569             kmo_dfs_print_parameter_help(parlist,
00570                                         "kmos.kmo_illumination.imethod"));
00571 
00572         neighborhoodRange = kmo_dfs_get_parameter_double(parlist,
00573                 "kmos.kmo_illumination.neighborhoodRange");
00574         KMO_TRY_CHECK_ERROR_STATE();
00575 
00576         KMO_TRY_ASSURE(neighborhoodRange > 0.0,
00577                 CPL_ERROR_ILLEGAL_INPUT,
00578                 "neighborhoodRange must be greater than 0.0");
00579 
00580         KMO_TRY_EXIT_IF_ERROR(
00581             kmo_dfs_print_parameter_help(parlist,
00582                                     "kmos.kmo_illumination.neighborhoodRange"));
00583 
00584         ranges_txt = kmo_dfs_get_parameter_string(parlist,
00585                                                   "kmos.kmo_illumination.range");
00586         KMO_TRY_CHECK_ERROR_STATE();
00587 
00588         KMO_TRY_EXIT_IF_ERROR(
00589             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.range"));
00590 
00591         ranges = kmo_identify_ranges(ranges_txt);
00592         KMO_TRY_CHECK_ERROR_STATE();
00593 
00594         flux = kmo_dfs_get_parameter_bool(parlist,
00595                                           "kmos.kmo_illumination.flux");
00596 
00597         KMO_TRY_ASSURE((flux == 0) ||
00598                        (flux == 1),
00599                        CPL_ERROR_ILLEGAL_INPUT,
00600                        "flux must be either FALSE or TRUE!");
00601 
00602         KMO_TRY_EXIT_IF_ERROR(
00603             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.flux"));
00604 
00605         add_all_sky = kmo_dfs_get_parameter_bool(parlist,
00606                                                  "kmos.kmo_illumination.add-all");
00607 
00608         KMO_TRY_ASSURE((add_all_sky == 0) ||
00609                        (add_all_sky == 1),
00610                        CPL_ERROR_ILLEGAL_INPUT,
00611                        "add_all must be either FALSE or TRUE!");
00612 
00613         KMO_TRY_EXIT_IF_ERROR(
00614             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.add-all"));
00615 
00616         pix_scale = kmo_dfs_get_parameter_double(parlist,
00617                                         "kmos.kmo_illumination.pix_scale");
00618         KMO_TRY_CHECK_ERROR_STATE();
00619         KMO_TRY_EXIT_IF_ERROR(
00620            kmo_dfs_print_parameter_help(parlist,
00621                                        "kmos.kmo_illumination.pix_scale"));
00622         KMO_TRY_ASSURE((pix_scale >= 0.01) &&
00623                        (pix_scale <= 0.4),
00624                        CPL_ERROR_ILLEGAL_INPUT,
00625                        "pix_scale must be between 0.01 and 0.4 (results in cubes "
00626                        "with 7x7 to 280x280 pixels)!");
00627 
00628         suppress_extension = kmo_dfs_get_parameter_bool(parlist,
00629                                           "kmos.kmo_illumination.suppress_extension");
00630         KMO_TRY_CHECK_ERROR_STATE();
00631         KMO_TRY_EXIT_IF_ERROR(
00632             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.suppress_extension"));
00633 
00634         KMO_TRY_ASSURE((suppress_extension == TRUE) || (suppress_extension == FALSE),
00635                        CPL_ERROR_ILLEGAL_INPUT,
00636                        "suppress_extension must be TRUE or FALSE!");
00637 
00638         kmo_band_pars_load(parlist, "kmos.kmo_illumination");
00639 
00640         KMO_TRY_EXIT_IF_ERROR(
00641             kmo_combine_pars_load(parlist,
00642                                   "kmos.kmo_illumination",
00643                                   &cmethod,
00644                                   &cpos_rej,
00645                                   &cneg_rej,
00646                                   &citer,
00647                                   &cmin,
00648                                   &cmax,
00649                                   FALSE));
00650         cpl_msg_info("", "-------------------------------------------");
00651 
00652         // check if filter_id, grating_id and rotator offset match for all
00653         // detectors
00654         KMO_TRY_EXIT_IF_ERROR(
00655             kmo_check_frameset_setup(frameset, FLAT_SKY,
00656                                        TRUE, FALSE, TRUE));
00657         KMO_TRY_EXIT_IF_ERROR(
00658             kmo_check_frame_setup(frameset, FLAT_SKY, XCAL,
00659                                        TRUE, FALSE, TRUE));
00660         KMO_TRY_EXIT_IF_ERROR(
00661             kmo_check_frame_setup(frameset, XCAL, YCAL,
00662                                        TRUE, FALSE, TRUE));
00663         KMO_TRY_EXIT_IF_ERROR(
00664             kmo_check_frame_setup(frameset, XCAL, LCAL,
00665                                        TRUE, FALSE, TRUE));
00666         KMO_TRY_EXIT_IF_ERROR(
00667             kmo_check_frame_setup(frameset, XCAL, MASTER_FLAT,
00668                                        TRUE, FALSE, TRUE));
00669 
00670         KMO_TRY_EXIT_IF_NULL(
00671             frame = kmo_dfs_get_frame(frameset, XCAL));
00672         KMO_TRY_EXIT_IF_NULL(
00673             suffix = kmo_dfs_get_suffix(frame, TRUE, FALSE));
00674 
00675         KMO_TRY_EXIT_IF_ERROR(
00676             kmo_check_frame_setup_md5_xycal(frameset));
00677         KMO_TRY_EXIT_IF_ERROR(
00678             kmo_check_frame_setup_md5(frameset));
00679 
00680         cpl_msg_info("", "Detected instrument setup:   %s", suffix+1);
00681         cpl_msg_info("", "(grating 1, 2 & 3)");
00682 
00683         // check which IFUs are active for all frames
00684         KMO_TRY_EXIT_IF_NULL(
00685             unused_ifus_before = kmo_get_unused_ifus(frameset, 0, 0));
00686 
00687         KMO_TRY_EXIT_IF_NULL(
00688             unused_ifus_after = kmo_duplicate_unused_ifus(unused_ifus_before));
00689 
00690         kmo_print_unused_ifus(unused_ifus_before, FALSE);
00691 
00692         // load desc for MASTER_DARK and check
00693         KMO_TRY_EXIT_IF_NULL(
00694             frame = kmo_dfs_get_frame(frameset, MASTER_DARK));
00695         desc_dark = kmo_identify_fits_header(
00696                     cpl_frame_get_filename(frame));
00697         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_DARK frame doesn't seem to "
00698                                       "be in KMOS-format!");
00699         KMO_TRY_ASSURE((desc_dark.nr_ext == 2*KMOS_NR_DETECTORS) &&
00700                        (desc_dark.ex_badpix == FALSE) &&
00701                        (desc_dark.fits_type == f2d_fits) &&
00702                        (desc_dark.frame_type == detector_frame),
00703                        CPL_ERROR_ILLEGAL_INPUT,
00704                        "MASTER_DARK isn't in the correct format!!!");
00705         nx = desc_dark.naxis1;
00706         ny = desc_dark.naxis2;
00707 
00708         // load desc for MASTER_FLAT and check
00709         KMO_TRY_EXIT_IF_NULL(
00710             frame = kmo_dfs_get_frame(frameset, MASTER_FLAT));
00711         desc_flat = kmo_identify_fits_header(cpl_frame_get_filename(frame));
00712         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_FLAT frame doesn't seem to "
00713                                       "be in KMOS-format!");
00714         KMO_TRY_ASSURE((desc_flat.nr_ext % (2*KMOS_NR_DETECTORS) == 0) &&
00715                        (desc_flat.ex_badpix == FALSE) &&
00716                        (desc_flat.fits_type == f2d_fits) &&
00717                        (desc_flat.frame_type == detector_frame),
00718                        CPL_ERROR_ILLEGAL_INPUT,
00719                        "MASTER_FLAT isn't in the correct format!!!");
00720 
00721         // load desc for XCAL and check
00722         KMO_TRY_EXIT_IF_NULL(
00723             xcalFrame = kmo_dfs_get_frame(frameset, XCAL));
00724         desc_xcal = kmo_identify_fits_header(cpl_frame_get_filename(xcalFrame));
00725         KMO_TRY_CHECK_ERROR_STATE_MSG("XCAL frame doesn't seem to "
00726                                       "be in KMOS-format!");
00727         KMO_TRY_ASSURE((desc_xcal.nr_ext % KMOS_NR_DETECTORS == 0) &&
00728                        (desc_xcal.ex_badpix == FALSE) &&
00729                        (desc_xcal.fits_type == f2d_fits) &&
00730                        (desc_xcal.frame_type == detector_frame),
00731                        CPL_ERROR_ILLEGAL_INPUT,
00732                        "XCAL isn't in the correct format!!!");
00733         KMO_TRY_ASSURE((desc_xcal.naxis1 == nx) &&
00734                        (desc_xcal.naxis2 == ny),
00735                        CPL_ERROR_ILLEGAL_INPUT,
00736                        "MASTER_DARK and XCAL frame haven't same dimensions! "
00737                        "(x,y): (%d,%d) vs (%d,%d)",
00738                        nx, ny, desc_xcal.naxis1, desc_xcal.naxis2);
00739 
00740         nr_devices = desc_xcal.nr_ext;
00741 
00742         // load desc for YCAL and check
00743         KMO_TRY_EXIT_IF_NULL(
00744             ycalFrame = kmo_dfs_get_frame(frameset, YCAL));
00745         desc_ycal = kmo_identify_fits_header(cpl_frame_get_filename(ycalFrame));
00746         KMO_TRY_CHECK_ERROR_STATE_MSG("YCAL frame doesn't seem to "
00747                                       "be in KMOS-format!");
00748         KMO_TRY_ASSURE((desc_ycal.nr_ext == desc_xcal.nr_ext) &&
00749                        (desc_ycal.ex_badpix == desc_xcal.ex_badpix) &&
00750                        (desc_ycal.fits_type == desc_xcal.fits_type) &&
00751                        (desc_ycal.frame_type == desc_xcal.frame_type),
00752                        CPL_ERROR_ILLEGAL_INPUT,
00753                        "YCAL isn't in the correct format!!!");
00754         KMO_TRY_ASSURE((desc_ycal.naxis1 == desc_xcal.naxis1) &&
00755                        (desc_ycal.naxis2 == desc_xcal.naxis2),
00756                        CPL_ERROR_ILLEGAL_INPUT,
00757                        "MASTER_DARK and YCAL frame haven't same dimensions! "
00758                        "(x,y): (%d,%d) vs (%d,%d)",
00759                        nx, ny, desc_ycal.naxis1, desc_ycal.naxis2);
00760 
00761         // load desc for LCAL and check
00762         KMO_TRY_EXIT_IF_NULL(
00763             lcalFrame = kmo_dfs_get_frame(frameset, LCAL));
00764         desc_lcal = kmo_identify_fits_header(cpl_frame_get_filename(lcalFrame));
00765         KMO_TRY_CHECK_ERROR_STATE_MSG("LCAL frame doesn't seem to "
00766                                       "be in KMOS-format!");
00767         KMO_TRY_ASSURE((desc_lcal.ex_badpix == desc_xcal.ex_badpix) &&
00768                        (desc_lcal.fits_type == desc_xcal.fits_type) &&
00769                        (desc_lcal.frame_type == desc_xcal.frame_type),
00770                        CPL_ERROR_ILLEGAL_INPUT,
00771                        "LCAL isn't in the correct format!!!");
00772         KMO_TRY_ASSURE((desc_lcal.naxis1 == desc_xcal.naxis1) &&
00773                        (desc_lcal.naxis2 == desc_xcal.naxis2),
00774                        CPL_ERROR_ILLEGAL_INPUT,
00775                        "MASTER_DARK and LCAL frame haven't same dimensions! "
00776                        "(x,y): (%d,%d) vs (%d,%d)",
00777                        nx, ny, desc_lcal.naxis1, desc_lcal.naxis2);
00778         KMO_TRY_EXIT_IF_NULL(
00779             tmp_header = kmo_dfs_load_primary_header(frameset, LCAL));
00780 
00781         // load desc for FLAT_SKY and check
00782         nr_devices = KMOS_NR_DETECTORS;
00783         KMO_TRY_EXIT_IF_NULL(
00784             frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00785 
00786         KMO_TRY_EXIT_IF_NULL(
00787             main_header = kmclipm_propertylist_load(
00788                                          cpl_frame_get_filename(frame), 0));
00789         rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00790         KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00791         kmclipm_strip_angle(&rotangle);
00792         cpl_propertylist_delete(main_header); main_header = NULL;
00793 
00794         cnt = 1;
00795         while (frame != NULL) {
00796             KMO_TRY_EXIT_IF_NULL(
00797                 main_header = kmclipm_propertylist_load(
00798                                              cpl_frame_get_filename(frame), 0));
00799 
00800             desc_sky = kmo_identify_fits_header(
00801                         cpl_frame_get_filename(frame));
00802             KMO_TRY_CHECK_ERROR_STATE_MSG("FLAT_SKY frame doesn't seem to "
00803                                           "be in KMOS-format!");
00804             KMO_TRY_ASSURE((desc_sky.nr_ext == 3) &&
00805                            (desc_sky.ex_badpix == FALSE) &&
00806                            (desc_sky.fits_type == raw_fits) &&
00807                            (desc_sky.frame_type == detector_frame),
00808                            CPL_ERROR_ILLEGAL_INPUT,
00809                            "FLAT_SKY isn't in the correct format!!!");
00810             KMO_TRY_ASSURE((desc_sky.naxis1 == nx) &&
00811                            (desc_sky.naxis2 == ny) &&
00812                            (desc_sky.nr_ext == nr_devices),
00813                            CPL_ERROR_ILLEGAL_INPUT,
00814                            "MASTER_DARK and FLAT_SKY (no. %d) frame haven't "
00815                            "same dimensions! (x,y): (%d,%d) vs (%d,%d)",
00816                            cnt, nx, ny, desc_flat.naxis1, desc_flat.naxis2);
00817             kmo_free_fits_desc(&desc_sky);
00818             kmo_init_fits_desc(&desc_sky);
00819 
00820             KMO_TRY_ASSURE(
00821                 (kmo_check_lamp(main_header, INS_LAMP1_ST) == FALSE) &&
00822                 (kmo_check_lamp(main_header, INS_LAMP2_ST) == FALSE) &&
00823                 (kmo_check_lamp(main_header, INS_LAMP3_ST) == FALSE) &&
00824                 (kmo_check_lamp(main_header, INS_LAMP4_ST) == FALSE),
00825                 CPL_ERROR_ILLEGAL_INPUT,
00826                 "All lamps must be switched off for the FLAT_SKY frames!");
00827 
00828             // assert that filters have correct IDs and that all detectors of
00829             // all input frames have the same filter set
00830             for (i = 1; i <= KMOS_NR_DETECTORS; i++) {
00831                 // ESO INS FILTi ID
00832                 KMO_TRY_EXIT_IF_NULL(
00833                     keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, i, IFU_FILTID_POSTFIX));
00834                 KMO_TRY_EXIT_IF_NULL(
00835                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00836 
00837                 KMO_TRY_EXIT_IF_NULL(
00838                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00839                 cpl_free(keyword); keyword = NULL;
00840 
00841                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00842                                (strcmp(filter_id, "YJ") == 0) ||
00843                                (strcmp(filter_id, "H") == 0) ||
00844                                (strcmp(filter_id, "K") == 0) ||
00845                                (strcmp(filter_id, "HK") == 0),
00846                                CPL_ERROR_ILLEGAL_INPUT,
00847                                "Filter ID in primary header must be either 'IZ', "
00848                                "'YJ', 'H', 'K' or " "'HK' !");
00849 
00850                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00851                                CPL_ERROR_ILLEGAL_INPUT,
00852                                "Filter IDs must be the same for FLAT_SKY frame"
00853                                " and lcal frame!"
00854                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00855                                i, cpl_frame_get_filename(frame),
00856                                filter_id, filter_id_l);
00857 
00858                 // ESO INS GRATi ID
00859                 KMO_TRY_EXIT_IF_NULL(
00860                     keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, i, IFU_GRATID_POSTFIX));
00861                 KMO_TRY_EXIT_IF_NULL(
00862                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00863 
00864                 KMO_TRY_EXIT_IF_NULL(
00865                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00866                 cpl_free(keyword); keyword = NULL;
00867 
00868                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00869                                (strcmp(filter_id, "YJ") == 0) ||
00870                                (strcmp(filter_id, "H") == 0) ||
00871                                (strcmp(filter_id, "K") == 0) ||
00872                                (strcmp(filter_id, "HK") == 0),
00873                                CPL_ERROR_ILLEGAL_INPUT,
00874                                "Grating ID in primary header must be either "
00875                                "'IZ', 'YJ', 'H', 'K' or " "'HK' !");
00876 
00877                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00878                                CPL_ERROR_ILLEGAL_INPUT,
00879                                "Grating IDs must be the same for FLAT_SKY frame"
00880                                " and lcal frame!"
00881                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00882                                i, cpl_frame_get_filename(frame),
00883                                filter_id, filter_id_l);
00884 
00885                 tmp_rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00886                 KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00887                 kmclipm_strip_angle(&tmp_rotangle);
00888                 KMO_TRY_ASSURE((abs(rotangle - tmp_rotangle) < 10.0) ||
00889                                (abs(rotangle - tmp_rotangle) > 360.-10.) ,
00890                         CPL_ERROR_ILLEGAL_INPUT,
00891                         "OCS ROT NAANGLE of sky flat frames differ too much: %f %f",
00892                         rotangle, tmp_rotangle);
00893             }
00894             cpl_propertylist_delete(main_header); main_header = NULL;
00895 
00896             // get next FLAT_SKY frame
00897             frame = kmo_dfs_get_frame(frameset, NULL);
00898             KMO_TRY_CHECK_ERROR_STATE();
00899             cnt++;
00900         }
00901 
00902         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00903 
00904         //
00905         // noise will be propagated when:
00906         // MASTER_DARK and MASTER_FLAT have noise extensions and if at least
00907         // 2 FLAT_SKY frames are provided.
00908         // Otherwise noise will be ignored.
00909         //
00910         if (desc_dark.ex_noise &&
00911             desc_flat.ex_noise &&
00912             (cpl_frameset_count_tags(frameset, FLAT_SKY) >= 2)) {
00913             process_noise = TRUE;
00914         }
00915 
00916         if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00917             cpl_msg_warning(cpl_func, "cmethod is changed to 'average' "
00918                             "since there is only one input frame! (The output "
00919                             "file won't have any noise extensions)");
00920 
00921             cmethod = "average";
00922         }
00923 
00924         //
00925         // Check whether 1st FLAT_SKY should be omitted
00926         //
00927         KMO_TRY_EXIT_IF_NULL(
00928             frameset_sky = cpl_frameset_new());
00929 
00930         if (add_all_sky) {
00931             // just add all FLAT_SKY frames without check
00932             frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00933             while (frame != NULL) {
00934                 KMO_TRY_EXIT_IF_ERROR(
00935                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00936                 frame = kmo_dfs_get_frame(frameset, NULL);
00937             }
00938             cpl_msg_info("", "Add all FLAT_SKY without checking for acquisition frame.");
00939         } else {
00940             // check if 1st FLAT_SKY has different exposure time and whether to omit it
00941             KMO_TRY_EXIT_IF_NULL(
00942                 frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00943 
00944             if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00945                 // just one FLAT_SKY, always add
00946                 KMO_TRY_EXIT_IF_ERROR(
00947                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00948                 KMO_TRY_CHECK_ERROR_STATE();
00949             } else {
00950                 // several FLAT_SKY frames, check exptime
00951 
00952                 // get exptime 1
00953                 KMO_TRY_EXIT_IF_NULL(
00954                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00955                 exptime1 = cpl_propertylist_get_double(main_header, EXPTIME);
00956                 KMO_TRY_CHECK_ERROR_STATE();
00957                 cpl_propertylist_delete(main_header); main_header = NULL;
00958 
00959                 // get exptime 2
00960                 frame = kmo_dfs_get_frame(frameset, NULL);
00961                 KMO_TRY_EXIT_IF_NULL(
00962                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00963                 exptime2 = cpl_propertylist_get_double(main_header, EXPTIME);
00964                 KMO_TRY_CHECK_ERROR_STATE();
00965                 cpl_propertylist_delete(main_header); main_header = NULL;
00966 
00967                 // loop remaining frames
00968                 same_exptime = TRUE;
00969                 frame = kmo_dfs_get_frame(frameset, NULL);
00970                 while (same_exptime && (frame != NULL)) {
00971                     KMO_TRY_EXIT_IF_NULL(
00972                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00973                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00974                     KMO_TRY_CHECK_ERROR_STATE();
00975                     cpl_propertylist_delete(main_header); main_header = NULL;
00976                     if (fabs(exptime-exptime2) > 0.01) {
00977                         // not same
00978                         same_exptime = FALSE;
00979                     }
00980                     frame = kmo_dfs_get_frame(frameset, NULL);
00981                 }
00982 
00983                 if (same_exptime) {
00984                     // frame [2,n] have same exptime, add them
00985                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00986                     KMO_TRY_EXIT_IF_NULL(
00987                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00988                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00989                     KMO_TRY_CHECK_ERROR_STATE();
00990                     cpl_propertylist_delete(main_header); main_header = NULL;
00991                     cpl_msg_info("", "Omit FLAT_SKY: %s with EXPTIME of %g sec (acquisition), other frame(s) have EXPTIME of %g sec", cpl_frame_get_filename(frame), exptime, exptime2);
00992                     frame = kmo_dfs_get_frame(frameset, NULL);
00993                     while (frame != NULL) {
00994                         KMO_TRY_EXIT_IF_ERROR(
00995                             cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00996                         frame = kmo_dfs_get_frame(frameset, NULL);
00997                     }
00998                     if (fabs(exptime1-exptime2) < 0.01) {
00999                         cpl_msg_warning("", "The 1st FLAT_SKY has the same exposure time as the following ones. "
01000                                             "It has anyway been omitted since we assume it is an acquisition frame. "
01001                                             "If you want to add it anyway call this recipe with the --add-all parameter");
01002                     }
01003                 } else {
01004                     cpl_msg_error("", "The exposure times of the FLAT_SKY frames don't match!");
01005                     cpl_msg_error("", "We assume that the 1st frame is an acquisition frame and would be omitted.");
01006                     cpl_msg_error("", "The following frames should have the same exposure time if they originate from the same template.");
01007                     cpl_msg_error("", "If you want to reduce them anyway call this recipe with the --add-all parameter");
01008                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
01009                     while (frame != NULL) {
01010                         KMO_TRY_EXIT_IF_NULL(
01011                             main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
01012                         exptime = cpl_propertylist_get_double(main_header, EXPTIME);
01013                         KMO_TRY_CHECK_ERROR_STATE();
01014                         cpl_propertylist_delete(main_header); main_header = NULL;
01015                         cpl_msg_error("", "FLAT_SKY: %s, EXPTIME: %g", cpl_frame_get_filename(frame), exptime);
01016                         frame = kmo_dfs_get_frame(frameset, NULL);
01017                     }
01018                     cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_INPUT);
01019                     KMO_TRY_CHECK_ERROR_STATE();
01020                 }
01021             }
01022         }
01023 
01024         KMO_TRY_EXIT_IF_NULL(
01025             frame = kmo_dfs_get_frame(frameset_sky, FLAT_SKY));
01026         KMO_TRY_EXIT_IF_NULL(
01027             main_header = kmo_dfs_load_primary_header(frameset_sky, FLAT_SKY));
01028         KMO_TRY_EXIT_IF_NULL(
01029             keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, 1, IFU_GRATID_POSTFIX));
01030         KMO_TRY_EXIT_IF_NULL(
01031             filter = cpl_sprintf("%s", cpl_propertylist_get_string(main_header, keyword)));
01032         cpl_free(keyword); keyword = NULL;
01033 
01034         //
01035         // set default band-specific ranges for collapsing
01036         //
01037         if (ranges == NULL) {
01038             if (strcmp(filter, "IZ") == 0) {
01039                 ranges_txt = "0.81,1.05";
01040             } else if (strcmp(filter, "YJ") == 0) {
01041                 ranges_txt = "1.025,1.3";
01042             } else if (strcmp(filter, "H") == 0) {
01043                 ranges_txt = "1.5,1.7";
01044             } else if (strcmp(filter, "K") == 0) {
01045                 ranges_txt = "2.1,2.35";
01046             } else if (strcmp(filter, "HK") == 0) {
01047                 ranges_txt = "1.5,1.7;2.1,2.35";
01048 //                ranges_txt = "1.5,1.7";
01049             } else {
01050                 KMO_TRY_ASSURE(1 == 0,
01051                                CPL_ERROR_ILLEGAL_INPUT,
01052                                "We really shouldn't get here...");
01053             }
01054             cpl_msg_info("", "Spectral range to collapse has been set to [%s] um for this band.", ranges_txt);
01055             ranges = kmo_identify_ranges(ranges_txt);
01056             KMO_TRY_CHECK_ERROR_STATE();
01057         }
01058 
01059         // setup grid definition, wavelength start and end points will be set
01060         // in the detector loop
01061         KMO_TRY_EXIT_IF_ERROR(
01062             kmclipm_setup_grid(&gd, method, neighborhoodRange, pix_scale, 0.));
01063 
01064         // create filename for LUT
01065         KMO_TRY_EXIT_IF_NULL(
01066             fn_lut = cpl_sprintf("%s%s", "lut", suffix));
01067 
01068         // extract bounds
01069         KMO_TRY_EXIT_IF_NULL(
01070             tmp_header = kmo_dfs_load_primary_header(frameset, XCAL));
01071         KMO_TRY_EXIT_IF_NULL(
01072             bounds = kmclipm_extract_bounds(tmp_header));
01073         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
01074 
01075         // get timestamps of xcal, ycal & lcal
01076         KMO_TRY_EXIT_IF_NULL(
01077             calTimestamp = kmo_get_timestamps(xcalFrame, ycalFrame, lcalFrame));
01078 
01079         // create arrays to hold reconstructed data and noise cubes and
01080         // their headers
01081         KMO_TRY_EXIT_IF_NULL(
01082             stored_data_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01083                                                             sizeof(cpl_imagelist*)));
01084         KMO_TRY_EXIT_IF_NULL(
01085             stored_noise_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01086                                                              sizeof(cpl_imagelist*)));
01087         KMO_TRY_EXIT_IF_NULL(
01088             stored_data_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01089                                                          sizeof(cpl_image*)));
01090         KMO_TRY_EXIT_IF_NULL(
01091             stored_noise_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01092                                                           sizeof(cpl_image*)));
01093         KMO_TRY_EXIT_IF_NULL(
01094             stored_sub_data_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01095                                                                      sizeof(cpl_propertylist*)));
01096         KMO_TRY_EXIT_IF_NULL(
01097             stored_sub_noise_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01098                                                                       sizeof(cpl_propertylist*)));
01099         KMO_TRY_EXIT_IF_NULL(
01100             edge_table_sky = (cpl_table***)cpl_calloc(KMOS_NR_DETECTORS,
01101                                                       sizeof(cpl_table**)));
01102         KMO_TRY_EXIT_IF_NULL(
01103             edge_table_flat = (cpl_table**)cpl_calloc(KMOS_IFUS_PER_DETECTOR,
01104                                                       sizeof(cpl_table*)));
01105         KMO_TRY_EXIT_IF_NULL(
01106             calAngles = cpl_vector_new(3));
01107 
01108         //
01109         // loop through all detectors
01110         //
01111         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01112             cpl_msg_info("","Processing detector No. %d", det_nr);
01113 
01114             KMO_TRY_EXIT_IF_NULL(
01115                 detector_in = cpl_imagelist_new());
01116 
01117             // load data of det_nr of all FLAT_SKY frames into an imagelist
01118             KMO_TRY_EXIT_IF_NULL(
01119                 img_in = kmo_dfs_load_image(frameset_sky, FLAT_SKY, det_nr, FALSE, TRUE, NULL));
01120 
01121             cnt = 0;
01122             while (img_in != NULL) {
01123                 cpl_imagelist_set(detector_in, img_in, cnt);
01124                 KMO_TRY_CHECK_ERROR_STATE();
01125 
01126                 /* load same extension of next FLAT_SKY frame*/
01127                 img_in = kmo_dfs_load_image(frameset_sky, NULL, det_nr, FALSE, TRUE, NULL);
01128                 KMO_TRY_CHECK_ERROR_STATE();
01129 
01130                 cnt++;
01131             }
01132 
01133             //
01134             // process imagelist
01135             //
01136 
01137             // combine imagelist (data only) and create noise (stdev of data)
01138             cpl_msg_info("","Combining frames...");
01139             if (process_noise) {
01140                 KMO_TRY_EXIT_IF_ERROR(
01141                     kmclipm_combine_frames(detector_in,
01142                                            NULL,
01143                                            NULL,
01144                                            cmethod,
01145                                            cpos_rej,
01146                                            cneg_rej,
01147                                            citer,
01148                                            cmax,
01149                                            cmin,
01150                                            &combined_data,
01151                                            &combined_noise,
01152                                            -1.0));
01153             } else {
01154                 KMO_TRY_EXIT_IF_ERROR(
01155                     kmclipm_combine_frames(detector_in,
01156                                            NULL,
01157                                            NULL,
01158                                            cmethod,
01159                                            cpos_rej,
01160                                            cneg_rej,
01161                                            citer,
01162                                            cmax,
01163                                            cmin,
01164                                            &combined_data,
01165                                            NULL,
01166                                            -1.0));
01167             }
01168 
01169             if (kmclipm_omit_warning_one_slice > 10) {
01170 // AA: commmented this out: Too unclear for the user, no benefit to know about this number
01171 //                cpl_msg_warning(cpl_func, "Previous warning (number of "
01172 //                                          "identified slices) occured %d times.",
01173 //                                kmclipm_omit_warning_one_slice);
01174                 kmclipm_omit_warning_one_slice = FALSE;
01175             }
01176 
01177             cpl_imagelist_delete(detector_in); detector_in = NULL;
01178 
01179             // load calibration files
01180             KMO_TRY_EXIT_IF_NULL(
01181                 xcal = kmo_dfs_load_cal_image(frameset, XCAL, det_nr, FALSE, rotangle,
01182                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01183 
01184             KMO_TRY_EXIT_IF_ERROR(
01185                 cpl_vector_set(calAngles, 0, rotangle_found));
01186             KMO_TRY_EXIT_IF_NULL(
01187                 ycal = kmo_dfs_load_cal_image(frameset, YCAL, det_nr, FALSE, rotangle,
01188                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01189             KMO_TRY_EXIT_IF_ERROR(
01190                 cpl_vector_set(calAngles, 1, rotangle_found));
01191             KMO_TRY_EXIT_IF_NULL(
01192                 lcal = kmo_dfs_load_cal_image(frameset, LCAL, det_nr, FALSE, rotangle,
01193                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01194             KMO_TRY_EXIT_IF_ERROR(
01195                 cpl_vector_set(calAngles, 2, rotangle_found));
01196 
01197             // load bad pixel mask from XCAL and set NaNs to 0 and all other values to 1
01198             KMO_TRY_EXIT_IF_NULL(
01199                 bad_pix_mask = cpl_image_duplicate(xcal));
01200 
01201             KMO_TRY_EXIT_IF_NULL(
01202                 pbad_pix_mask = cpl_image_get_data_float(bad_pix_mask));
01203             for (x = 0; x < nx; x++) {
01204                 for (y = 0; y < ny; y++) {
01205                     if (isnan(pbad_pix_mask[x+nx*y])) {
01206                         pbad_pix_mask[x+nx*y] = 0.;
01207                     } else {
01208                         pbad_pix_mask[x+nx*y] = 1.;
01209                     }
01210                 }
01211             }
01212             KMO_TRY_CHECK_ERROR_STATE();
01213 
01214             //
01215             // calculate SKYFLAT_EDGE
01216             //
01217             if (has_flat_edge) {
01218                 // get edge-edgepars from FLAT_SKY
01219                 KMO_TRY_EXIT_IF_ERROR(
01220                     kmo_calc_edgepars(combined_data,
01221                                       unused_ifus_after[det_nr-1],
01222                                       bad_pix_mask,
01223                                       det_nr,
01224                                       &slitlet_ids,
01225                                       &edgepars));
01226                 KMO_TRY_CHECK_ERROR_STATE();
01227 
01228                 // copy edgepars to table for saving later on
01229                 KMO_TRY_EXIT_IF_NULL(
01230                     edge_table_sky[det_nr-1] = kmo_edgepars_to_table(slitlet_ids, edgepars));
01231 
01232                 if (edgepars != NULL) {
01233                     for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01234                         cpl_matrix_delete(edgepars[i]); edgepars[i] = NULL;
01235                     }
01236                     cpl_free(edgepars); edgepars = NULL;
01237                 }
01238                 if (slitlet_ids != NULL) {
01239                     for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01240                         cpl_vector_delete(slitlet_ids[i]); slitlet_ids[i] = NULL;
01241                     }
01242                     cpl_free(slitlet_ids); slitlet_ids = NULL;
01243                 }
01244                 KMO_TRY_CHECK_ERROR_STATE();
01245 
01246                 //
01247                 // correlate FLAT_EDGE and SKYFLAT_EDGE
01248                 //
01249 
01250                 // load flat_edge from MASTER_FLAT
01251                 KMO_TRY_EXIT_IF_NULL(
01252                     frame = kmo_dfs_get_frame(frameset, FLAT_EDGE));
01253                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01254                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01255 
01256                     KMO_TRY_EXIT_IF_NULL(
01257                         punused_ifus = cpl_array_get_data_int_const(unused_ifus_after[det_nr-1]));
01258                     if (punused_ifus[j] == 0) {
01259                         KMO_TRY_EXIT_IF_NULL(
01260                             edge_table_flat[j] = kmclipm_cal_table_load(cpl_frame_get_filename(frame),
01261                                                                         ifu_nr, rotangle, 0, &tmp_rotangle));
01262                     }
01263                 }
01264 
01265                 //
01266                 // calculate shift value
01267                 //
01268 
01269                 KMO_TRY_EXIT_IF_NULL(
01270                     shift_vec = cpl_vector_new(KMOS_IFUS_PER_DETECTOR));
01271                 KMO_TRY_EXIT_IF_NULL(
01272                     edge_vec = cpl_vector_new(2*KMOS_SLITLET_X));
01273 
01274                 // get shift values for each IFU by comparing all edge parameters,
01275                 // rejecting and applying median
01276                 int row = 1024; // middle of frame
01277                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01278                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01279                     for (edgeNr = 0; edgeNr < 2*KMOS_SLITLET_X; edgeNr++) {
01280                         if (edge_table_flat[j] != NULL) {
01281                             double flatval = kmo_calc_fitted_slitlet_edge(edge_table_flat[j], edgeNr, row);
01282                             double skyval  = kmo_calc_fitted_slitlet_edge(edge_table_sky[det_nr-1][j], edgeNr, row);
01283                             cpl_vector_set(edge_vec, edgeNr, flatval-skyval);
01284                         }
01285                     }
01286 
01287                     // reject deviating edge-differences
01288                     kmclipm_vector *kv = NULL;
01289                     KMO_TRY_EXIT_IF_NULL(
01290                         kv = kmclipm_vector_create(cpl_vector_duplicate(edge_vec)));
01291                     kmclipm_reject_deviant(kv, 3, 3, NULL, NULL);
01292 
01293                     // set shift value for each IFU
01294                     cpl_vector_set(shift_vec, j, kmclipm_vector_get_median(kv, KMCLIPM_ARITHMETIC));
01295                     kmclipm_vector_delete(kv); kv = NULL;
01296                 }
01297                 cpl_vector_delete(edge_vec); edge_vec = NULL;
01298                 KMO_TRY_CHECK_ERROR_STATE();
01299 
01300                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01301                     cpl_table_delete(edge_table_flat[j]);
01302                     edge_table_flat[j] = NULL;
01303                 }
01304 
01305                 // take median of all IFU-shift-values
01306                 double shift_val = -cpl_vector_get_median(shift_vec);
01307                 cpl_vector_delete(shift_vec); shift_vec = NULL;
01308 
01309                 cpl_msg_info("", "Shift detector %d by %g pixels.", det_nr, shift_val);
01310 
01311                 int     xdim                = cpl_image_get_size_x(combined_data),
01312                         ydim                = cpl_image_get_size_y(combined_data);
01313                 double  *array_in           = cpl_calloc(xdim, sizeof(double)),
01314                         *array_out          = NULL;
01315                 float   *pcombined_data     = cpl_image_get_data_float(combined_data),
01316                         *pcombined_noise    = NULL;
01317     //            float   *tmpArray           = cpl_calloc(xdim, sizeof(float));
01318                 if (process_noise) {
01319                     pcombined_noise = cpl_image_get_data_float(combined_noise);
01320                 }
01321 
01322                 for (iy = 0; iy < ydim; iy++) {
01323                     // cubic spline
01324                     for (ix = 0; ix < xdim; ix++) {
01325                         array_in[ix] = pcombined_data[ix+iy*xdim];
01326                     }
01327                     array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01328                                                     xdim, shift_val, 1.0,
01329                                                     NATURAL);
01330                     for (ix = 0; ix < xdim; ix++) {
01331                       pcombined_data[ix+iy*xdim] = array_out[ix];
01332                     }
01333                     cpl_free(array_out);
01334 
01335     //                // linear
01336     //                for (ix = 1; ix < xdim; ix++) {
01337     //                    tmpArray[ix-1] = (pcombined_data[ix+iy*xdim]-pcombined_data[(ix-1)+iy*xdim])*shift_val +
01338     //                                     pcombined_data[(ix-1)+iy*xdim];
01339     //                }
01340     //                for (ix = 1; ix < xdim; ix++) {
01341     //                    pcombined_data[ix+iy*xdim] = tmpArray[ix];
01342     //                }
01343 
01344                     if (process_noise) {
01345                         // cubic spline
01346                         for (ix = 0; ix < xdim; ix++) {
01347                             array_in[ix] = pcombined_noise[ix+iy*xdim];
01348                         }
01349                         array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01350                                                         xdim, shift_val, 1.0,
01351                                                         NATURAL);
01352                         for (ix = 0; ix < xdim; ix++) {
01353                           pcombined_noise[ix+iy*xdim] = array_out[ix];
01354                         }
01355                         cpl_free(array_out);
01356 
01357     //                    // linear
01358     //                    for (ix = 1; ix < xdim; ix++) {
01359     //                        tmpArray[ix-1] = (pcombined_noise[ix+iy*xdim]-pcombined_noise[(ix-1)+iy*xdim])*shift_val +
01360     //                                         pcombined_noise[(ix-1)+iy*xdim];
01361     //                    }
01362     //                    for (ix = 1; ix < xdim; ix++) {
01363     //                        pcombined_noise[ix+iy*xdim] = tmpArray[ix];
01364     //                    }
01365                     }
01366                 }
01367                 cpl_free(array_in); array_in = NULL;
01368             }
01369             //
01370             // reconstruct
01371             //
01372             // load MASTER_DARK and MASTER_FLAT
01373             KMO_TRY_EXIT_IF_NULL(
01374                 img_dark = kmo_dfs_load_image(frameset, MASTER_DARK,
01375                                               det_nr, FALSE, FALSE, NULL));
01376 
01377             if (process_noise) {
01378                 KMO_TRY_EXIT_IF_NULL(
01379                     img_dark_noise = kmo_dfs_load_image(frameset, MASTER_DARK,
01380                                                         det_nr, TRUE, FALSE, NULL));
01381             }
01382 
01383             KMO_TRY_EXIT_IF_NULL(
01384                 img_flat = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, FALSE,
01385                                                   rotangle, FALSE, NULL,
01386                                                   &rotangle_found, -1, 0, 0));
01387 
01388             if (process_noise) {
01389                 KMO_TRY_EXIT_IF_NULL(
01390                     img_flat_noise = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, TRUE,
01391                                                             rotangle, FALSE, NULL,
01392                                                             &rotangle_found, -1, 0, 0));
01393             }
01394 
01395             char *tmp_band_method = getenv("KMO_BAND_METHOD");
01396             int band_method = 0;
01397             if (tmp_band_method != NULL) {
01398                 band_method = atoi(tmp_band_method);
01399             }
01400 
01401             // ESO INS FILTi ID
01402             KMO_TRY_EXIT_IF_NULL(
01403                 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, det_nr,
01404                                       IFU_FILTID_POSTFIX));
01405             KMO_TRY_EXIT_IF_NULL(
01406                 filter_id = cpl_propertylist_get_string(main_header, keyword));
01407             cpl_free(keyword); keyword = NULL;
01408 
01409             KMO_TRY_EXIT_IF_NULL(
01410                 band_table = kmo_dfs_load_table(frameset, WAVE_BAND, 1, 0));
01411             KMO_TRY_EXIT_IF_ERROR(
01412                 kmclipm_setup_grid_band_lcal(&gd, lcal, filter_id, band_method,
01413                                              band_table));
01414             cpl_table_delete(band_table); band_table = NULL;
01415 
01416             cpl_msg_info("","Reconstructing cubes...");
01417             for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01418                 // update sub-header
01419                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01420 
01421                 // load raw image and sub-header
01422                 KMO_TRY_EXIT_IF_NULL(
01423                     sub_header = kmo_dfs_load_sub_header(frameset_sky, FLAT_SKY,
01424                                                          det_nr, FALSE));
01425 
01426                 KMO_TRY_EXIT_IF_NULL(
01427                     punused_ifus = cpl_array_get_data_int_const(
01428                                                   unused_ifus_after[det_nr-1]));
01429 
01430                 // check if IFU is valid according to main header keywords &
01431                 // calibration files
01432                 KMO_TRY_EXIT_IF_NULL(
01433                     keyword = cpl_sprintf("%s%d%s", IFU_VALID_PREFIX, ifu_nr,
01434                                           IFU_VALID_POSTFIX));
01435                 KMO_TRY_CHECK_ERROR_STATE();
01436                 ranges_txt = cpl_propertylist_get_string(main_header, keyword);
01437                 cpl_free(keyword); keyword = NULL;
01438 
01439                 if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) &&
01440                     (bounds[2*(ifu_nr-1)] != -1) &&
01441                     (bounds[2*(ifu_nr-1)+1] != -1) &&
01442                     (punused_ifus[j] == 0))
01443                 {
01444                     // IFU is valid
01445                     cpl_error_reset();
01446 
01447                     // calculate WCS
01448                     KMO_TRY_EXIT_IF_ERROR(
01449                         kmo_calc_wcs_gd(main_header, sub_header, ifu_nr, gd));
01450 
01451                     // reconstruct data
01452                     KMO_TRY_EXIT_IF_ERROR(
01453                         kmo_reconstruct_sci_image(ifu_nr,
01454                                                 bounds[2*(ifu_nr-1)],
01455                                                 bounds[2*(ifu_nr-1)+1],
01456                                                 combined_data,
01457                                                 combined_noise,
01458                                                 img_dark,
01459                                                 img_dark_noise,
01460                                                 img_flat,
01461                                                 img_flat_noise,
01462                                                 xcal,
01463                                                 ycal,
01464                                                 lcal,
01465                                                 &gd,
01466                                                 calTimestamp,
01467                                                 calAngles,
01468                                                 fn_lut,
01469                                                 &cube_data,
01470                                                 &cube_noise,
01471                                                 flux,
01472                                                 background,
01473                                                 NULL,
01474                                                 NULL,
01475                                                 NULL));
01476                     KMO_TRY_CHECK_ERROR_STATE();
01477                 } else {
01478                     // IFU is invalid
01479                     cpl_error_reset();
01480                 } // if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) ...
01481 
01482                 // save output
01483                 KMO_TRY_EXIT_IF_NULL(
01484                     extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_DATA));
01485 
01486                 KMO_TRY_EXIT_IF_ERROR(
01487                     kmclipm_update_property_string(sub_header, EXTNAME,
01488                                                    extname,
01489                                                    "FITS extension name"));
01490 
01491                 cpl_free(extname); extname = NULL;
01492 
01493                 // store cube and sub header into array for later
01494                 stored_data_cubes[ifu_nr - 1] = cube_data;
01495                 stored_sub_data_headers[ifu_nr - 1] = sub_header;
01496 
01497                 if (process_noise) {
01498                     KMO_TRY_EXIT_IF_NULL(
01499                         sub_header = cpl_propertylist_duplicate(
01500                                            stored_sub_data_headers[ifu_nr - 1]));
01501                     KMO_TRY_EXIT_IF_NULL(
01502                         extname = kmo_extname_creator(ifu_frame, ifu_nr,
01503                                                       EXT_NOISE));
01504 
01505                     KMO_TRY_EXIT_IF_ERROR(
01506                         kmclipm_update_property_string(sub_header,
01507                                                 EXTNAME,
01508                                                 extname,
01509                                                 "FITS extension name"));
01510 
01511                     cpl_free(extname); extname = NULL;
01512 
01513                     stored_noise_cubes[ifu_nr - 1] = cube_noise;
01514                     stored_sub_noise_headers[ifu_nr - 1] = sub_header;
01515                 }
01516                 cpl_image_delete(data_ifu); data_ifu = NULL;
01517                 cpl_image_delete(noise_ifu); noise_ifu = NULL;
01518                 cube_data = NULL;
01519                 cube_noise = NULL;
01520             } // for j IFUs
01521 
01522             // free memory
01523             cpl_image_delete(combined_data); combined_data = NULL;
01524             cpl_image_delete(combined_noise); combined_noise = NULL;
01525             cpl_image_delete(xcal); xcal = NULL;
01526             cpl_image_delete(ycal); ycal = NULL;
01527             cpl_image_delete(lcal); lcal = NULL;
01528             cpl_image_delete(img_dark); img_dark = NULL;
01529             cpl_image_delete(img_flat); img_flat = NULL;
01530             cpl_image_delete(bad_pix_mask); bad_pix_mask = NULL;
01531             if (process_noise) {
01532                 cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01533                 cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01534             }
01535         } // for nr_devices
01536 
01537         cpl_free(edge_table_flat); edge_table_flat = NULL;
01538 
01539         // collapse cubes using rejection
01540         cpl_msg_info("","Collapsing cubes...");
01541         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01542             for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01543                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01544 
01545                 KMO_TRY_EXIT_IF_NULL(
01546                     punused_ifus = cpl_array_get_data_int_const(
01547                                                   unused_ifus_after[det_nr-1]));
01548                 if (punused_ifus[j] == 0) {
01549                     if (stored_sub_data_headers[ifu_nr-1] != NULL) {
01550                         // IFU is valid
01551                         ifu_crpix = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01552                                                                 CRPIX3);
01553                         KMO_TRY_CHECK_ERROR_STATE_MSG(
01554                                        "CRPIX3 keyword in FITS-header is missing!");
01555 
01556                         ifu_crval = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01557                                                                 CRVAL3);
01558                         KMO_TRY_CHECK_ERROR_STATE_MSG(
01559                                        "CRVAL3 keyword in FITS-header is missing!");
01560 
01561                         ifu_cdelt = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01562                                                                 CDELT3);
01563                         KMO_TRY_CHECK_ERROR_STATE_MSG(
01564                                        "CDELT3 keyword in FITS-header is missing!");
01565 
01566                         KMO_TRY_EXIT_IF_NULL(
01567                             identified_slices = kmo_identify_slices(ranges,
01568                                                                     ifu_crpix,
01569                                                                     ifu_crval,
01570                                                                     ifu_cdelt,
01571                                                                     gd.l.dim));
01572                     }/* else {
01573                         KMO_TRY_EXIT_IF_NULL(
01574                             identified_slices = cpl_vector_new(gd.l.dim));
01575                         cpl_vector_fill(identified_slices, 1.);
01576                     }*/
01577 
01578                     if (stored_data_cubes[ifu_nr-1] != NULL) {
01579                         KMO_TRY_EXIT_IF_ERROR(
01580                             kmclipm_make_image(stored_data_cubes[ifu_nr-1],
01581                                                stored_noise_cubes[ifu_nr-1],
01582                                                &stored_data_images[ifu_nr-1],
01583                                                &stored_noise_images[ifu_nr-1],
01584                                                identified_slices,
01585                                                cmethod, cpos_rej, cneg_rej,
01586                                                citer, cmax, cmin));
01587                     }
01588                     cpl_vector_delete(identified_slices); identified_slices = NULL;
01589                 } else {
01590                     // IFU is invalid
01591                 }
01592             }
01593         }
01594 
01595         // normalise all IFUs of a detector as a group.
01596         // Calculate mean of each IFU, add up and divide by number of successful
01597         // averaged IFUs.
01598         // Then divide all valid IFUs with mean value
01599         int jj;
01600         for (jj = 0; jj < nr_devices; jj++) {
01601             cnt = 0;
01602             mean_data = 0;
01603             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01604                 ifu_nr = jj*KMOS_IFUS_PER_DETECTOR + i;
01605                 if (stored_data_images[ifu_nr] != NULL) {
01606                     KMO_TRY_ASSURE(cpl_image_count_rejected(stored_data_images[ifu_nr]) <
01607                                    cpl_image_get_size_x(stored_data_images[ifu_nr])*
01608                                    cpl_image_get_size_y(stored_data_images[ifu_nr]),
01609                                    CPL_ERROR_ILLEGAL_INPUT,
01610                                    "The collapsed, dark-subtracted image contains "
01611                                    "only invalid values! Probably the provided "
01612                                    "FLAT_SKY frames are exactly the same as the "
01613                                    "frames used for MASTER_DARK calculation.");
01614 
01615                     mean_data += cpl_image_get_mean(stored_data_images[ifu_nr]);
01616                     KMO_TRY_CHECK_ERROR_STATE();
01617                     cnt++;
01618                 }
01619 
01620             }
01621             mean_data /= cnt;
01622 
01623             if (mean_data != 0.0) {
01624                 for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01625                     ifu_nr = jj*KMOS_IFUS_PER_DETECTOR + i;
01626                     if (stored_data_images[ifu_nr] != NULL) {
01627                         KMO_TRY_EXIT_IF_ERROR(
01628                             cpl_image_divide_scalar(stored_data_images[ifu_nr],
01629                                                     mean_data));
01630                     }
01631                 }
01632             } else {
01633                 cpl_msg_warning(cpl_func, "Data couldn't be normalised "
01634                                           "(mean = 0.0)!");
01635             }
01636 
01637             if (process_noise) {
01638                 if (mean_data != 0.0) {
01639                     for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01640                         ifu_nr = jj*KMOS_IFUS_PER_DETECTOR + i;
01641                         if (stored_noise_images[ifu_nr] != NULL) {
01642                             KMO_TRY_EXIT_IF_ERROR(
01643                                 cpl_image_divide_scalar(stored_noise_images[ifu_nr],
01644                                                         mean_data));
01645                         }
01646                     }
01647                 } else {
01648                     cpl_msg_warning(cpl_func, "Noise couldn't be normalised "
01649                                               "(mean = 0.0)!");
01650                 }
01651             }
01652         } // end for(jj)
01653 
01654         // calculate qc parameters on normalised data
01655         qc_spat_unif = 0.0;
01656         cnt = 0;
01657         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01658             if (stored_data_images[i] != NULL) {
01659                 tmp_mean = cpl_image_get_mean(stored_data_images[i]);
01660                 tmp_stdev = cpl_image_get_stdev (stored_data_images[i]);
01661 
01662                 qc_spat_unif += pow(tmp_mean-1, 2);
01663                 if (fabs(tmp_mean) > qc_max_dev) {
01664                     qc_max_dev = tmp_mean-1;
01665                     qc_max_dev_id = i+1;
01666                 }
01667                 if (fabs(tmp_stdev) > qc_max_nonunif) {
01668                     qc_max_nonunif = tmp_stdev;
01669                     qc_max_nonunif_id = i+1;
01670                 }
01671                 KMO_TRY_CHECK_ERROR_STATE();
01672                 cnt++;
01673             }
01674         }
01675         qc_spat_unif = sqrt(qc_spat_unif / cnt);
01676 
01677         //
01678         // save data
01679         //
01680 
01681         // update which IFUs are not used
01682         kmo_print_unused_ifus(unused_ifus_after, TRUE);
01683 
01684         KMO_TRY_EXIT_IF_ERROR(
01685             kmo_set_unused_ifus(unused_ifus_after, main_header,
01686                                 "kmo_illumination"));
01687 
01688         cpl_msg_info("","Saving data...");
01689 
01690         KMO_TRY_EXIT_IF_ERROR(
01691             kmclipm_update_property_double(main_header, QC_SPAT_UNIF, qc_spat_unif,
01692                                            "[adu] uniformity of illumination correction"));
01693         KMO_TRY_EXIT_IF_ERROR(
01694             kmclipm_update_property_double(main_header, QC_SPAT_MAX_DEV, qc_max_dev,
01695                                            "[adu] max. deviation from unity"));
01696         KMO_TRY_EXIT_IF_ERROR(
01697             kmclipm_update_property_int(main_header, QC_SPAT_MAX_DEV_ID, qc_max_dev_id,
01698                                         "[] IFU ID with max. dev. from unity"));
01699         KMO_TRY_EXIT_IF_ERROR(
01700             kmclipm_update_property_double(main_header, QC_SPAT_MAX_NONUNIF, qc_max_nonunif,
01701                                            "[adu] max. stdev of illumination corr."));
01702         KMO_TRY_EXIT_IF_ERROR(
01703             kmclipm_update_property_int(main_header, QC_SPAT_MAX_NONUNIF_ID, qc_max_nonunif_id,
01704                                         "[] IFU ID with max. stdev in illum. corr."));
01705 
01706         if (!suppress_extension) {
01707             KMO_TRY_EXIT_IF_NULL(
01708                 fn_suffix = cpl_sprintf("%s", suffix));
01709         } else {
01710             KMO_TRY_EXIT_IF_NULL(
01711                 fn_suffix = cpl_sprintf("%s", ""));
01712         }
01713         KMO_TRY_EXIT_IF_ERROR(
01714             kmo_dfs_save_main_header(frameset, ILLUM_CORR, fn_suffix, frame,
01715                                      main_header, parlist, cpl_func));
01716 
01717         if (has_flat_edge) {
01718             KMO_TRY_EXIT_IF_ERROR(
01719                 kmo_dfs_save_main_header(frameset, SKYFLAT_EDGE, fn_suffix, frame,
01720                                          main_header, parlist, cpl_func));
01721         }
01722 
01723         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01724             KMO_TRY_EXIT_IF_ERROR(
01725                 kmo_dfs_save_image(stored_data_images[i], ILLUM_CORR, fn_suffix,
01726                                    stored_sub_data_headers[i], 0./0.));
01727 
01728             if (process_noise) {
01729                 KMO_TRY_EXIT_IF_ERROR(
01730                     kmo_dfs_save_image(stored_noise_images[i], ILLUM_CORR,
01731                                        fn_suffix, stored_sub_noise_headers[i], 0./0.));
01732             }
01733         }
01734 
01735         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
01736             for (ifu_nr = 0; ifu_nr < KMOS_IFUS_PER_DETECTOR; ifu_nr++) {
01737                 KMO_TRY_EXIT_IF_ERROR(
01738                     kmclipm_update_property_int(stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01739                                                 CAL_IFU_NR,
01740                                                 ifu_nr+1+(det_nr-1)*KMOS_IFUS_PER_DETECTOR,
01741                                                 "IFU Number {1..24}"));
01742                 KMO_TRY_EXIT_IF_ERROR(
01743                     kmclipm_update_property_double(
01744                                                 stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01745                                                 CAL_ROTANGLE,
01746                                                 rotangle_found,
01747                                                 "[deg] Rotator relative to nasmyth"));
01748                 if (has_flat_edge) {
01749                     // save edge-parameters as product
01750                     KMO_TRY_EXIT_IF_ERROR(
01751                         kmo_dfs_save_table(edge_table_sky[det_nr-1][ifu_nr], SKYFLAT_EDGE, fn_suffix,
01752                                            stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr]));
01753                 }
01754             }
01755         }
01756     }
01757     KMO_CATCH
01758     {
01759         KMO_CATCH_MSG();
01760         ret_val = -1;
01761     }
01762     kmo_free_fits_desc(&desc_sky);
01763     kmo_free_fits_desc(&desc_dark);
01764     kmo_free_fits_desc(&desc_flat);
01765     kmo_free_fits_desc(&desc_xcal);
01766     kmo_free_fits_desc(&desc_ycal);
01767     kmo_free_fits_desc(&desc_lcal);
01768     cpl_image_delete(combined_data); combined_data = NULL;
01769     cpl_image_delete(combined_noise); combined_noise = NULL;
01770     cpl_image_delete(xcal); xcal = NULL;
01771     cpl_image_delete(ycal); ycal = NULL;
01772     cpl_image_delete(lcal); lcal = NULL;
01773     cpl_image_delete(img_dark); img_dark = NULL;
01774     cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01775     cpl_image_delete(img_flat); img_flat = NULL;
01776     cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01777     cpl_array_delete(calTimestamp); calTimestamp = NULL;
01778     cpl_free(bounds); bounds = NULL;
01779     kmo_free_unused_ifus(unused_ifus_before); unused_ifus_before = NULL;
01780     kmo_free_unused_ifus(unused_ifus_after); unused_ifus_after = NULL;
01781     cpl_free(fn_lut); fn_lut = NULL;
01782     cpl_free(suffix); suffix = NULL;
01783     cpl_free(fn_suffix); fn_suffix = NULL;
01784     cpl_frameset_delete(frameset_sky); frameset_sky = NULL;
01785     cpl_vector_delete(ranges); ranges = NULL;
01786     cpl_free(filter); filter = NULL;
01787     if (calAngles != NULL) {
01788         cpl_vector_delete(calAngles); calAngles = NULL;
01789     }
01790     cpl_propertylist_delete(main_header); main_header = NULL;
01791     for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01792         if (stored_data_cubes != NULL) {
01793             cpl_imagelist_delete(stored_data_cubes[i]);
01794             stored_data_cubes[i] = NULL;
01795         }
01796         if (stored_noise_cubes != NULL) {
01797             cpl_imagelist_delete(stored_noise_cubes[i]);
01798             stored_noise_cubes[i] = NULL;
01799         }
01800         if (stored_data_images != NULL) {
01801             cpl_image_delete(stored_data_images[i]);
01802             stored_data_images[i] = NULL;
01803         }
01804         if (stored_noise_images != NULL) {
01805             cpl_image_delete(stored_noise_images[i]);
01806             stored_noise_images[i] = NULL;
01807         }
01808         if (stored_sub_data_headers != NULL) {
01809             cpl_propertylist_delete(stored_sub_data_headers[i]);
01810             stored_sub_data_headers[i] = NULL;
01811         }
01812         if (stored_sub_noise_headers != NULL) {
01813             cpl_propertylist_delete(stored_sub_noise_headers[i]);
01814             stored_sub_noise_headers[i] = NULL;
01815         }
01816     }
01817     cpl_free(stored_data_cubes); stored_data_cubes = NULL;
01818     cpl_free(stored_noise_cubes); stored_noise_cubes = NULL;
01819     cpl_free(stored_data_images); stored_data_images = NULL;
01820     cpl_free(stored_noise_images); stored_noise_images = NULL;
01821     cpl_free(stored_sub_data_headers); stored_sub_data_headers = NULL;
01822     cpl_free(stored_sub_noise_headers); stored_sub_noise_headers = NULL;
01823     if (edge_table_sky != NULL) {
01824         for (i = 0; i < KMOS_NR_DETECTORS; i++) {
01825             if (edge_table_sky[i] != NULL) {
01826                 for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01827                     cpl_table_delete(edge_table_sky[i][j]);
01828                     edge_table_sky[i][j] = NULL;
01829                 }
01830                 cpl_free(edge_table_sky[i]); edge_table_sky[i] = NULL;
01831             }
01832         }
01833         cpl_free(edge_table_sky); edge_table_sky = NULL;
01834     }
01835     if (edge_table_flat != NULL) {
01836         for (j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01837             cpl_table_delete(edge_table_flat[j]);
01838             edge_table_flat[j] = NULL;
01839         }
01840         cpl_free(edge_table_flat); edge_table_flat = NULL;
01841     }
01842 
01843     return ret_val;
01844 }
01845