KMOS Pipeline Reference Manual  1.2.7
kmo_combine.c
00001 /* $Id: kmo_combine.c,v 1.32 2013-06-27 13:10:21 aagudo Exp $
00002  *
00003  * This file is part of the KMOS Pipeline
00004  * Copyright (C) 2002,2003 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: aagudo $
00023  * $Date: 2013-06-27 13:10:21 $
00024  * $Revision: 1.32 $
00025  * $Name: not supported by cvs2svn $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 #include <string.h>
00033 #include <math.h>
00034 
00035 #include <cpl.h>
00036 #include <cpl_wcs.h>
00037 
00038 #include "kmo_debug.h"
00039 #include "kmo_utils.h"
00040 #include "kmo_dfs.h"
00041 #include "kmo_error.h"
00042 #include "kmo_priv_functions.h"
00043 #include "kmo_cpl_extensions.h"
00044 #include "kmo_constants.h"
00045 #include "kmo_priv_combine.h"
00046 
00047 static int kmo_combine_create(cpl_plugin *);
00048 static int kmo_combine_exec(cpl_plugin *);
00049 static int kmo_combine_destroy(cpl_plugin *);
00050 static int kmo_combine(cpl_parameterlist *, cpl_frameset *);
00051 
00052 static char kmo_combine_description[] =
00053 "This recipe shifts several exposures of an object and combines them. The diffe-\n"
00054 "rent methods to match the exposures are described below (--method parameter).\n"
00055 "The output cube is larger than the input cubes, according to the shifts to be\n"
00056 "applied. Additionally a border of NaN values is added. The WCS is the same as\n"
00057 "for the first exposure.\n"
00058 "For each spatial/spectral pixel a new value will be calculated (according the\n"
00059 "--cmethod parameter) and written into the output cube.\n"
00060 "Only exposures with equal orientation regarding the WCS can be combined (except\n"
00061 "-–method=”none”), north must point to the same direction. It is recommended to\n"
00062 "apply any rotation possibly after combining.\n"
00063 "\n"
00064 "The behavior of the selection of IFUs to combine differs for some templates and\n"
00065 "can be controlled with the parameters --name and --ifus.\n"
00066 "If the input data cubes stem from templates KMOS_spec_obs_mapping8 or\n"
00067 "KMOS_spec_obs_mapping24 all extensions from all input frames are combined into\n"
00068 "a single map by default (like in recipe kmo_sci_red). If just the area of a\n"
00069 "specific IFU should be combined, the parameter --ifus can be specified, or more\n"
00070 "easily --name.\n"
00071 "If the input data cubes stem from other templates like e.g.\n"
00072 "KMOS_spec_obs_freedither all extensions of all input frames are combined into\n"
00073 "several output frames by default. The input IFUs are grouped according their\n"
00074 "targeted object name stored in the keywords ESO OCS ARMx NAME. If just a\n"
00075 "specific object should be combined, its name can be specified with parameter\n"
00076 "--name. If arbitrary IFUs shoukd be comined, one can specify these with the\n"
00077 "parameter --ifus.\n"
00078 "\n"
00079 "The default mapping mode is done via the --name parameter, where the name of\n"
00080 "the object has to be provided. The recipe searches in all input data cubes IFUs\n"
00081 "pointing to that object.\n"
00082 "\n"
00083 "BASIC PARAMETERS:\n"
00084 "-----------------\n"
00085 "--name\n"
00086 "--ifus\n"
00087 "Since an object can be present only once per exposure and since it can be\n"
00088 "located in different IFUs for the existing exposures, there are two modes to\n"
00089 "identify the objects:\n"
00090 "   * Combine by object names (default)\n"
00091 "   In this case the object name must be provided via the --name parameter. The\n"
00092 "   object name will be searched for in all primary headers of all provided frames\n"
00093 "   in the keyword ESO OCS ARMx NAME.\n"
00094 "\n"
00095 "   * Combine by index (advanced)\n"
00096 "   In this case the --ifus parameter must be provided. The parameter must have\n"
00097 "   the same number of entries as frames are provided, e.g. \"3;1;24\" for 3 expo-\n"
00098 "   sures. The index doesn't reference the extension in the frame but the real\n"
00099 "   index of the IFU as defined in the EXTNAME keyword (e.g. 'IFU.3.DATA').\n"
00100 "\n"
00101 "--method\n"
00102 "There are following sources to get the shift parameters from:\n"
00103 "   * 'none' (default)\n"
00104 "   The cubes are directly recombined, not shifting at all. The ouput frame will\n"
00105 "   have the same dimensions as the input cubes.\n"
00106 "   If the size differs a warning will be emitted and the cubes will be aligned\n"
00107 "   to the lower left corner. If the orientation differs a warning will be emit-\n"
00108 "   ted, but the cubes are combined anyway.\n"
00109 "\n"
00110 "   * 'header'\n"
00111 "   The shifts are calculated according to the WCS information stored in the\n"
00112 "   header of every IFU. The output frame will get larger, except the object is\n"
00113 "   at the exact same position for all exposures. The size of the exposures can\n"
00114 "   differ, but the orientation must be the same for all exposures.\n"
00115 "\n"
00116 "   * 'center'\n"
00117 "   The shifts are calculated using a centering algorithm. The cube will be col-\n"
00118 "   lapsed and a 2D profile will be fitted to it to identify the centre. With \n"
00119 "   the parameter --fmethod the function to fit can be provided. The size of the\n"
00120 "   exposures can differ, but the orientation must be the same for all exposures.\n"
00121 "\n"
00122 "   * 'user'\n"
00123 "   Read the shifts from a user specified file. The path of the file must be pro-\n"
00124 "   vided using the --filename parameter. For every exposure (except the first one)\n"
00125 "   two shift values are expected per line, they have to be separated with simple\n"
00126 "   spaces. The values indicate pixel shifts and are referenced to the first\n"
00127 "   frame. The 1st value is the shift in x-direction to the left, the 2nd the\n"
00128 "   shift in y-direction upwards. The size of the exposures can differ, but the\n"
00129 "   orientation must be the same for all exposures.\n"
00130 "\n"
00131 "--cmethod\n"
00132 "Following methods of frame combination are available:\n"
00133 "   * 'ksigma' (Default)\n"
00134 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00135 "   are examined. If they deviate significantly, they will be rejected according\n"
00136 "   to the conditions:\n"
00137 "       val > mean + stdev * cpos_rej\n"
00138 "   and\n"
00139 "       val < mean - stdev * cneg_rej\n"
00140 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00141 "   parameters. In the first iteration median and percentile level are used.\n"
00142 "\n"
00143 "   * 'median'\n"
00144 "   At each pixel position the median is calculated.\n"
00145 "\n"
00146 "   * 'average'\n"
00147 "   At each pixel position the average is calculated.\n"
00148 "\n"
00149 "   * 'sum'\n"
00150 "   At each pixel position the sum is calculated.\n"
00151 "\n"
00152 "   * 'min_max'\n"
00153 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00154 "   --cmax and --cmin apply to this method.\n"
00155 "\n"
00156 "ADVANCED PARAMETERS\n"
00157 "-------------------\n"
00158 "--edge_nan\n"
00159 "Set borders of two sides of the cubes to NaN before combining them. This minimises\n"
00160 "unwanted border effects when dithering.\n"
00161 "\n"
00162 "--fmethod\n"
00163 "see --method='center'\n"
00164 "The type of function that should be fitted spatially to the collapsed image.\n"
00165 "This fit is used to create a mask to extract the spectrum of the object. Valid\n"
00166 "values are “gauss” and “moffat”.\n"
00167 "\n"
00168 "--filename\n"
00169 "see --method='user'\n"
00170 "\n"
00171 "--cpos_rej\n"
00172 "--cneg_rej\n"
00173 "--citer\n"
00174 "see --cmethod='ksigma'\n"
00175 "\n"
00176 "--cmax\n"
00177 "--cmin\n"
00178 "see --cmethod='min_max'\n"
00179 "\n"
00180 "--flux\n"
00181 "Specify if flux conservation should be applied.\n"
00182 "\n"
00183 "--suppress_extension\n"
00184 "If set to TRUE, the arbitrary filename extensions are supressed. If multiple\n"
00185 "products with the same category are produced, they will be numered consecutively\n"
00186 "starting from 0.\n"
00187 "\n"
00188 "-------------------------------------------------------------------------------\n"
00189 "  Input files:\n"
00190 "\n"
00191 "   DO                      KMOS                                                \n"
00192 "   category                Type   Explanation                  Required #Frames\n"
00193 "   --------                -----  -----------                  -------- -------\n"
00194 "   <none or any>           F3I    data frame                       Y      2-n  \n"
00195 "\n"
00196 "  Output files:\n"
00197 "\n"
00198 "   DO                      KMOS\n"
00199 "   category                Type   Explanation\n"
00200 "   --------                -----  -----------\n"
00201 "   COMBINE_<ESO PRO CATG>  F3I    Combined data cube\n"
00202 "   EXP_MASK_<ESO PRO CATG> F3I    Exposure time mask\n"
00203 "-------------------------------------------------------------------------------\n"
00204 "\n";
00205 
00222 int cpl_plugin_get_info(cpl_pluginlist *list)
00223 {
00224     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00225     cpl_plugin *plugin = &recipe->interface;
00226 
00227     cpl_plugin_init(plugin,
00228                         CPL_PLUGIN_API,
00229                         KMOS_BINARY_VERSION,
00230                         CPL_PLUGIN_TYPE_RECIPE,
00231                         "kmo_combine",
00232                         "Combine reconstructed cubes",
00233                         kmo_combine_description,
00234                         "Alex Agudo Berbel",
00235                         "kmos-spark@mpe.mpg.de",
00236                         kmos_get_license(),
00237                         kmo_combine_create,
00238                         kmo_combine_exec,
00239                         kmo_combine_destroy);
00240 
00241     cpl_pluginlist_append(list, plugin);
00242 
00243     return 0;
00244 }
00245 
00253 static int kmo_combine_create(cpl_plugin *plugin)
00254 {
00255     cpl_recipe *recipe;
00256     cpl_parameter *p;
00257 
00258     /* Check that the plugin is part of a valid recipe */
00259     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00260         recipe = (cpl_recipe *)plugin;
00261     else
00262         return -1;
00263 
00264     /* Create the parameters list in the cpl_recipe object */
00265     recipe->parameters = cpl_parameterlist_new();
00266 
00267     /* Fill the parameters list */
00268     /* --name */
00269     p = cpl_parameter_new_value("kmos.kmo_combine.name",
00270                                 CPL_TYPE_STRING,
00271                                 "Name of the object to combine.",
00272                                 "kmos.kmo_combine",
00273                                 "");
00274     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "name");
00275     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00276     cpl_parameterlist_append(recipe->parameters, p);
00277 
00278     /* --ifus */
00279     p = cpl_parameter_new_value("kmos.kmo_combine.ifus",
00280                                 CPL_TYPE_STRING,
00281                                 "The indices of the IFUs to combine. "
00282                                 "\"ifu1;ifu2;...\"",
00283                                 "kmos.kmo_combine",
00284                                 "");
00285     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "ifus");
00286     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00287     cpl_parameterlist_append(recipe->parameters, p);
00288 
00289     /* --method */
00290     p = cpl_parameter_new_value("kmos.kmo_combine.method",
00291                                 CPL_TYPE_STRING,
00292                                 "The shifting method:   "
00293                                 "'none': no shifting, combined directly "
00294                                                                   "(default), "
00295                                 "'header': shift according to WCS, "
00296                                 "'center': centering algorithm, "
00297                                 "'user': read shifts from file",
00298                                 "kmos.kmo_combine",
00299                                 "none");
00300     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "method");
00301     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00302     cpl_parameterlist_append(recipe->parameters, p);
00303 
00304     /* --fmethod */
00305     p = cpl_parameter_new_value("kmos.kmo_combine.fmethod",
00306                                 CPL_TYPE_STRING,
00307                                 "The fitting method (applies only when "
00308                                 "method='center'):   "
00309                                 "'gauss': fit a gauss function to collapsed "
00310                                 "image (default), "
00311                                 "'moffat': fit a moffat function to collapsed"
00312                                 " image",
00313                                 "kmos.kmo_combine",
00314                                 "gauss");
00315     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "fmethod");
00316     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00317     cpl_parameterlist_append(recipe->parameters, p);
00318 
00319     /* --filename */
00320     p = cpl_parameter_new_value("kmos.kmo_combine.filename",
00321                                 CPL_TYPE_STRING,
00322                                 "The path to the file with the shift vectors."
00323                                 "(Applies only to method='user')",
00324                                 "kmos.kmo_combine",
00325                                 "");
00326     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "filename");
00327     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00328     cpl_parameterlist_append(recipe->parameters, p);
00329 
00330     /* --flux */
00331     p = cpl_parameter_new_value("kmos.kmo_combine.flux",
00332                                 CPL_TYPE_BOOL,
00333                                 "Apply flux conservation: "
00334                                 "(TRUE (apply) or "
00335                                 "FALSE (don't apply)",
00336                                 "kmos.kmo_combine",
00337                                 FALSE);
00338     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00339     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00340     cpl_parameterlist_append(recipe->parameters, p);
00341 
00342     /* --edge_nan */
00343     p = cpl_parameter_new_value("kmos.kmo_combine.edge_nan",
00344                                 CPL_TYPE_BOOL,
00345                                 "Set borders of cubes to NaN before combining them."
00346                                 "(TRUE (apply) or "
00347                                 "FALSE (don't apply)",
00348                                 "kmos.kmo_combine",
00349                                 FALSE);
00350     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "edge_nan");
00351     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00352     cpl_parameterlist_append(recipe->parameters, p);
00353 
00354     /* --suppress_extension */
00355     p = cpl_parameter_new_value("kmos.kmo_combine.suppress_extension",
00356                                 CPL_TYPE_BOOL,
00357                                 "Suppress arbitrary filename extension."
00358                                 "(TRUE (apply) or FALSE (don't apply)",
00359                                 "kmos.kmo_combine",
00360                                 FALSE);
00361     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "suppress_extension");
00362     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00363     cpl_parameterlist_append(recipe->parameters, p);
00364 
00365     return kmo_combine_pars_create(recipe->parameters,
00366                                    "kmos.kmo_combine",
00367                                    DEF_REJ_METHOD,
00368                                    FALSE);
00369 }
00370 
00376 static int kmo_combine_exec(cpl_plugin *plugin)
00377 {
00378     cpl_recipe  *recipe;
00379 
00380     /* Get the recipe out of the plugin */
00381     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00382         recipe = (cpl_recipe *)plugin;
00383     else return -1 ;
00384 
00385     return kmo_combine(recipe->parameters, recipe->frames);
00386 }
00387 
00393 static int kmo_combine_destroy(cpl_plugin *plugin)
00394 {
00395     cpl_recipe *recipe;
00396 
00397     /* Get the recipe out of the plugin */
00398     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00399         recipe = (cpl_recipe *)plugin;
00400     else return -1 ;
00401 
00402     cpl_parameterlist_delete(recipe->parameters);
00403     return 0 ;
00404 }
00405 
00420 static int kmo_combine(cpl_parameterlist *parlist, cpl_frameset *frameset)
00421 {
00422     int              ret_val             = 0,
00423                      nr_frames           = 0,
00424                      index               = 0,
00425                      data_cube_counter   = 0,
00426                      noise_cube_counter  = 0,
00427                      citer               = 0,
00428                      cmin                = 0,
00429                      cmax                = 0,
00430                      flux                = FALSE,
00431                      edge_nan            = FALSE,
00432                      name_vec_size       = 0,
00433                      found               = 0,
00434                      suppress_extension  = FALSE,
00435                      suppress_index      = 0,
00436                      i                   = 0,
00437                      j                   = 0,
00438                      ifu_nr              = 0,
00439                      nv                  = 0;
00440     double           cpos_rej            = 0.0,
00441                      cneg_rej            = 0.0;
00442     char             *tmp_str            = NULL,
00443                      *fn_combine         = NULL,
00444                      *fn_mask            = NULL,
00445                      *mapping_mode       = NULL,
00446                      *name               = NULL,
00447                      **name_vec          = NULL;
00448     const char       *method             = NULL,
00449                      *cmethod            = NULL,
00450                      *fmethod            = NULL,
00451                      *filename           = NULL,
00452                      *frame_filename     = NULL,
00453                      *ifus_txt           = NULL,
00454                      *tmp_strc           = NULL;
00455     cpl_vector       *ifus               = NULL;
00456     cpl_image        *exp_mask           = NULL;
00457     cpl_imagelist    **data_cube_list    = NULL,
00458                      **noise_cube_list   = NULL,
00459                      *cube_combined_data = NULL,
00460                      *cube_combined_noise= NULL;
00461     cpl_propertylist *main_header        = NULL,
00462                      **data_header_list  = NULL,
00463                      **noise_header_list = NULL,
00464                      *tmp_header         = NULL;
00465     cpl_frame        *frame              = NULL;
00466     cpl_size         ci                  = 0;
00467     main_fits_desc   desc;
00468     enum extrapolationType extrapol_enum = NONE_CLIPPING;
00469 
00470     KMO_TRY
00471     {
00472         /* --- check input --- */
00473         KMO_TRY_ASSURE((parlist != NULL) &&
00474                        (frameset != NULL),
00475                        CPL_ERROR_NULL_INPUT,
00476                        "Not all input data is provided!");
00477 
00478         nr_frames = cpl_frameset_get_size(frameset);
00479 
00480         KMO_TRY_ASSURE(nr_frames >= 2,
00481                        CPL_ERROR_NULL_INPUT,
00482                        "At least two frames must be provided to combine!");
00483 
00484         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_combine") == 1,
00485                        CPL_ERROR_ILLEGAL_INPUT,
00486                        "Cannot identify RAW and CALIB frames!");
00487 
00488         cpl_msg_info("", "--- Parameter setup for kmo_combine -------");
00489 
00490         KMO_TRY_EXIT_IF_NULL(
00491             method = kmo_dfs_get_parameter_string(parlist,
00492                                            "kmos.kmo_combine.method"));
00493 
00494         KMO_TRY_EXIT_IF_NULL(
00495             fmethod = kmo_dfs_get_parameter_string(parlist,
00496                                            "kmos.kmo_combine.fmethod"));
00497 
00498         KMO_TRY_ASSURE((strcmp(method, "none") == 0) ||
00499                        (strcmp(method, "header") == 0) ||
00500                        (strcmp(method, "center") == 0) ||
00501                        (strcmp(method, "user") == 0),
00502                        CPL_ERROR_ILLEGAL_INPUT,
00503                        "Following shift methods are available : 'none', "
00504                        "'header', 'center' or 'user'");
00505 
00506         if (strcmp(method, "user") == 0) {
00507             filename = kmo_dfs_get_parameter_string(parlist,
00508                                                    "kmos.kmo_combine.filename");
00509             KMO_TRY_CHECK_ERROR_STATE();
00510 
00511             KMO_TRY_ASSURE(strcmp(filename, "") != 0,
00512                            CPL_ERROR_ILLEGAL_INPUT,
00513                            "path of file with shift information must be "
00514                            "provided!");
00515 
00516             KMO_TRY_EXIT_IF_ERROR(
00517                 kmo_dfs_print_parameter_help(parlist,
00518                                              "kmos.kmo_combine.filename"));
00519         }
00520 
00521         KMO_TRY_EXIT_IF_ERROR(
00522             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_combine.method"));
00523 
00524         ifus_txt = kmo_dfs_get_parameter_string(parlist,
00525                                                   "kmos.kmo_combine.ifus");
00526         KMO_TRY_CHECK_ERROR_STATE();
00527 
00528         name = (char*)kmo_dfs_get_parameter_string(parlist, "kmos.kmo_combine.name");
00529         KMO_TRY_CHECK_ERROR_STATE();
00530 
00531         if (strcmp(ifus_txt, "") != 0) {
00532             KMO_TRY_ASSURE(strcmp(name, "") == 0,
00533                            CPL_ERROR_ILLEGAL_INPUT,
00534                            "name parameter must be NULL if IFU indices are "
00535                            "provided!");
00536 
00537             KMO_TRY_EXIT_IF_NULL(
00538                 ifus = kmo_identify_values(ifus_txt));
00539 
00540             KMO_TRY_ASSURE(cpl_vector_get_size(ifus) == nr_frames,
00541                            CPL_ERROR_ILLEGAL_INPUT,
00542                            "ifus parameter must have the same number of values "
00543                            "than frames provided ) (%lld!=%d)",
00544                            cpl_vector_get_size(ifus), nr_frames);
00545         }
00546 
00547         if (strcmp(name, "") != 0) {
00548             KMO_TRY_ASSURE(strcmp(ifus_txt, "") == 0,
00549                            CPL_ERROR_ILLEGAL_INPUT,
00550                            "ifus parameter must be NULL if name is provided!");
00551         }
00552 
00553         flux = kmo_dfs_get_parameter_bool(parlist,
00554                                           "kmos.kmo_combine.flux");
00555         KMO_TRY_CHECK_ERROR_STATE();
00556         KMO_TRY_EXIT_IF_ERROR(
00557             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_combine.flux"));
00558 
00559         KMO_TRY_ASSURE((flux == TRUE) || (flux == FALSE),
00560                        CPL_ERROR_ILLEGAL_INPUT,
00561                        "flux must be TRUE or FALSE!");
00562 
00563         edge_nan = kmo_dfs_get_parameter_bool(parlist,
00564                                           "kmos.kmo_combine.edge_nan");
00565         KMO_TRY_CHECK_ERROR_STATE();
00566         KMO_TRY_EXIT_IF_ERROR(
00567             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_combine.edge_nan"));
00568 
00569         KMO_TRY_ASSURE((edge_nan == TRUE) || (edge_nan == FALSE),
00570                        CPL_ERROR_ILLEGAL_INPUT,
00571                        "edge_nan must be TRUE or FALSE!");
00572 
00573         suppress_extension = kmo_dfs_get_parameter_bool(parlist,
00574                                           "kmos.kmo_combine.suppress_extension");
00575         KMO_TRY_CHECK_ERROR_STATE();
00576         KMO_TRY_EXIT_IF_ERROR(
00577             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_combine.suppress_extension"));
00578 
00579         KMO_TRY_ASSURE((suppress_extension == TRUE) || (suppress_extension == FALSE),
00580                        CPL_ERROR_ILLEGAL_INPUT,
00581                        "suppress_extension must be TRUE or FALSE!");
00582 
00583         KMO_TRY_EXIT_IF_ERROR(
00584             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_combine.ifus"));
00585 
00586         KMO_TRY_EXIT_IF_ERROR(
00587             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_combine.name"));
00588 
00589         KMO_TRY_EXIT_IF_ERROR(
00590             kmo_combine_pars_load(parlist,
00591                                   "kmos.kmo_combine",
00592                                   &cmethod,
00593                                   &cpos_rej,
00594                                   &cneg_rej,
00595                                   &citer,
00596                                   &cmin,
00597                                   &cmax,
00598                                   FALSE));
00599 
00600         cpl_msg_info("", "-------------------------------------------");
00601 
00602         // load data and noise
00603         KMO_TRY_EXIT_IF_NULL(
00604             data_cube_list = cpl_calloc(nr_frames*KMOS_NR_IFUS,
00605                                         sizeof(cpl_imagelist*)));
00606 
00607         KMO_TRY_EXIT_IF_NULL(
00608             data_header_list = cpl_calloc(nr_frames*KMOS_NR_IFUS,
00609                                           sizeof(cpl_propertylist*)));
00610 
00611         KMO_TRY_EXIT_IF_NULL(
00612             noise_cube_list = cpl_calloc(nr_frames*KMOS_NR_IFUS,
00613                                          sizeof(cpl_imagelist*)));
00614 
00615         KMO_TRY_EXIT_IF_NULL(
00616             noise_header_list = cpl_calloc(nr_frames*KMOS_NR_IFUS,
00617                                            sizeof(cpl_propertylist*)));
00618 
00619         //
00620         // check for mapping mode
00621         //
00622         cpl_size fs_size = cpl_frameset_get_size(frameset);
00623         KMO_TRY_CHECK_ERROR_STATE();
00624 
00625         for (ci = 0; ci < fs_size; ci++) {
00626             KMO_TRY_EXIT_IF_NULL(
00627                 frame = cpl_frameset_get_position(frameset, ci));
00628 
00629             KMO_TRY_EXIT_IF_NULL(
00630                 tmp_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00631             if (cpl_propertylist_has(tmp_header, TPL_ID)) {
00632                 KMO_TRY_EXIT_IF_NULL(
00633                     tmp_strc = cpl_propertylist_get_string(tmp_header, TPL_ID));
00634                 if (mapping_mode == NULL) {
00635                     if (strcmp(tmp_strc, MAPPING8) == 0)
00636                     {
00637                         mapping_mode = cpl_sprintf("%s", tmp_strc);
00638                     }
00639                     if (strcmp(tmp_strc, MAPPING24) == 0)
00640                     {
00641                         mapping_mode = cpl_sprintf("%s", tmp_strc);
00642                     }
00643                 } else {
00644                     if (strcmp(tmp_strc, mapping_mode) != 0)
00645                     {
00646                         cpl_msg_warning("","There are different TPL IDs present in "
00647                                            "the set of frames: %s and %s",
00648                                            tmp_strc, mapping_mode);
00649                     }
00650                 }
00651             }
00652             cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00653         }
00654 
00655         if (mapping_mode != NULL) {
00656             if ((strcmp(ifus_txt, "") == 0)  && (strcmp(name, "") == 0)) {
00657                 cpl_msg_info("","**************************************************");
00658                 cpl_msg_info("","*  A map containing all IFUs will be generated!  *");
00659                 cpl_msg_info("","**************************************************");
00660                 extrapol_enum = BCS_NATURAL;
00661             } else {
00662                 cpl_msg_info("","The frames aren't combined into a map although they originate "
00663                                 "from a mapping template. But since the name- or ifu-parameter "
00664                                 "has been specified, the default behaviour is overridden.");
00665                 cpl_free(mapping_mode); mapping_mode = NULL;
00666             }
00667         }
00668 
00669         //
00670         // create name/ifu map...
00671         //
00672         KMO_TRY_EXIT_IF_NULL(
00673             name_vec = cpl_calloc(nr_frames*KMOS_NR_IFUS, sizeof(char*)));
00674 
00675         if ((strcmp(ifus_txt, "") == 0) &&
00676             (strcmp(name, "") == 0) &&
00677             (mapping_mode == NULL))
00678         {
00679             // all available names should be combined in one go
00680             name_vec_size = 0;
00681             for (i = 0; i < nr_frames; i++) {
00682                 KMO_TRY_EXIT_IF_NULL(
00683                     tmp_str = cpl_sprintf("%d", i));
00684                 KMO_TRY_EXIT_IF_NULL(
00685                     frame = kmo_dfs_get_frame(frameset, tmp_str));
00686                 cpl_free(tmp_str); tmp_str = NULL;
00687 
00688                 for (ifu_nr = 1; ifu_nr <= KMOS_NR_IFUS; ifu_nr++) {
00689                     found = 0;
00690                     tmp_str = kmo_get_name_from_ocs_ifu(frame, ifu_nr);
00691                     KMO_TRY_CHECK_ERROR_STATE();
00692 
00693                     if (tmp_str != NULL) {
00694                         for (j = 0; j < name_vec_size; j++) {
00695                             if (strcmp(name_vec[j], tmp_str) == 0) {
00696                                 found = TRUE;
00697                                 break;
00698                             }
00699                         }
00700                         if (!found) {
00701                             name_vec[name_vec_size++] = tmp_str;
00702                         } else {
00703                             cpl_free(tmp_str); tmp_str = NULL;
00704                         }
00705                     }
00706                 }
00707             }
00708         } else {
00709             // standard behavior: either ifu_nr- or name- or mapping-case
00710             name_vec_size = 1;
00711             if (mapping_mode != NULL) {
00712                 KMO_TRY_EXIT_IF_NULL(
00713                     name_vec[0] = cpl_sprintf("mapping"));
00714             } else {
00715                 if (ifus != NULL) {
00716                     char *tmptmp = NULL;
00717                     if (strlen(ifus_txt) > 10) {
00718                         KMO_TRY_EXIT_IF_NULL(
00719                             tmptmp = kmo_shorten_ifu_string(ifus_txt));
00720                         cpl_msg_info("", "Because of lengthy ifus-parameter, filenames of products and their EXTNAME keywords will be truncated to ...ifu%s...", tmptmp);
00721                     } else {
00722                         KMO_TRY_EXIT_IF_NULL(
00723                             tmptmp = cpl_sprintf("%s", ifus_txt));
00724                     }
00725                     KMO_TRY_EXIT_IF_NULL(
00726                         name_vec[0] = cpl_sprintf("IFU%s", tmptmp));
00727                     ifus_txt = "";
00728                     cpl_free(tmptmp); tmptmp = NULL;
00729                 } else {
00730                     KMO_TRY_EXIT_IF_NULL(
00731                         name_vec[0] = cpl_sprintf("%s", name));
00732                 }
00733             }
00734         }
00735 
00736         //
00737         // load all data (and noise if existent) cubes and store them
00738         //
00739         for (nv = 0; nv < name_vec_size; nv++){
00740             name = name_vec[nv];
00741 
00742             data_cube_counter = 0;
00743             noise_cube_counter = 0;
00744             for (i = 0; i < nr_frames; i++) {
00745                 KMO_TRY_EXIT_IF_NULL(
00746                     tmp_str = cpl_sprintf("%d", i));
00747 
00748                 KMO_TRY_EXIT_IF_NULL(
00749                     frame = kmo_dfs_get_frame(frameset, tmp_str));
00750 
00751                 KMO_TRY_EXIT_IF_NULL(
00752                     frame_filename = cpl_frame_get_filename(frame));
00753 
00754                 kmo_init_fits_desc(&desc);
00755 
00756                 desc = kmo_identify_fits_header(frame_filename);
00757                 KMO_TRY_CHECK_ERROR_STATE_MSG("Provided fits file doesn't seem to "
00758                                               "be in KMOS-format!");
00759 
00760                 KMO_TRY_ASSURE(desc.fits_type == f3i_fits,
00761                                CPL_ERROR_ILLEGAL_INPUT,
00762                                "Frame No. %d hasn't correct data type "
00763                                "(must be of type F3I)!", i+1);
00764 
00765                 if (mapping_mode != NULL) {
00766                     // we are in mapping mode
00767                     for (j = 1; j <= KMOS_NR_IFUS; j++) {
00768                         //loop over all IFUs
00769                         if (desc.sub_desc[j-1].valid_data == TRUE) {
00770                             // load data frames
00771                             override_err_msg = TRUE;
00772                             data_cube_list[data_cube_counter] =
00773                                 kmo_dfs_load_cube(frameset, tmp_str, j, FALSE);
00774                             override_err_msg = FALSE;
00775                             if (data_cube_list[data_cube_counter] == NULL) {
00776                                 // no data found for this IFU
00777                                 cpl_error_reset();
00778                             } else {
00779                                 if (edge_nan) {
00780                                     KMO_TRY_EXIT_IF_ERROR(
00781                                         kmo_edge_nan(data_cube_list[data_cube_counter], j));
00782                                 }
00783                                 KMO_TRY_EXIT_IF_NULL(
00784                                     data_header_list[data_cube_counter] =
00785                                         kmo_dfs_load_sub_header(frameset, tmp_str, j, FALSE));
00786                                 cpl_propertylist_update_string(data_header_list[data_cube_counter],
00787                                                         "ESO PRO FRNAME",
00788                                                         frame_filename);
00789                                 cpl_propertylist_update_int(data_header_list[data_cube_counter],
00790                                                         "ESO PRO IFUNR",
00791                                                         j);
00792                                 data_cube_counter++;
00793                             }
00794 
00795                             // load noise frames
00796                             override_err_msg = TRUE;
00797                             noise_cube_list[noise_cube_counter] =
00798                                 kmo_dfs_load_cube(frameset, tmp_str, j, TRUE);
00799 
00800                             override_err_msg = FALSE;
00801                             if (noise_cube_list[noise_cube_counter] == NULL) {
00802                                 // no noise found for this IFU
00803                                 cpl_error_reset();
00804                             } else {
00805                                 if (edge_nan) {
00806                                     KMO_TRY_EXIT_IF_ERROR(
00807                                         kmo_edge_nan(noise_cube_list[noise_cube_counter], j));
00808                                 }
00809                                 KMO_TRY_EXIT_IF_NULL(
00810                                     noise_header_list[noise_cube_counter] =
00811                                         kmo_dfs_load_sub_header(frameset, tmp_str, j, TRUE));
00812                                 noise_cube_counter++;
00813                             }
00814 
00815                             // check for every iteration if number of data and noise
00816                             // frames is the same
00817                             if (noise_cube_counter > 0) {
00818                                 KMO_TRY_ASSURE(data_cube_counter == noise_cube_counter,
00819                                                CPL_ERROR_ILLEGAL_INPUT,
00820                                                "Frame No. %d (%s) has no noise frame "
00821                                                "while the preceeding ones had!",
00822                                                i+1, frame_filename);
00823                             }
00824                         } // end if valid_data
00825                     }
00826                 } else {
00827                     // we are in name/ifu mode (single)
00828                     if (ifus != NULL) {
00829                         ifu_nr = cpl_vector_get(ifus, i);
00830                         KMO_TRY_CHECK_ERROR_STATE();
00831                     } else {
00832                         ifu_nr = kmo_get_index_from_ocs_name(frame, name);
00833                         KMO_TRY_CHECK_ERROR_STATE();
00834                     }
00835 
00836                     if (ifu_nr > 0) {
00837                         index = kmo_identify_index(frame_filename, ifu_nr , FALSE);
00838                         KMO_TRY_CHECK_ERROR_STATE();
00839 
00840                         if (desc.sub_desc[index-1].valid_data == TRUE) {
00841                             // load data frames
00842                             override_err_msg = TRUE;
00843                             data_cube_list[data_cube_counter] =
00844                                 kmo_dfs_load_cube(frameset, tmp_str, ifu_nr, FALSE);
00845                             override_err_msg = FALSE;
00846                             if (data_cube_list[data_cube_counter] == NULL) {
00847                                 // no data found for this IFU
00848                                 cpl_error_reset();
00849                                 if (ifus != NULL) {
00850                                     cpl_msg_warning(cpl_func, "Frame No. %d (%s) "
00851                                                     "doesn't contain IFU No. %d!", i+1,
00852                                                     frame_filename, ifu_nr);
00853                                 } else {
00854                                     cpl_msg_warning(cpl_func, "Frame No. %d (%s) "
00855                                                     "doesn't contain IFU with object "
00856                                                     "name '%s'!", i+1,
00857                                                     frame_filename, name);
00858                                 }
00859                             } else {
00860                                 if (edge_nan) {
00861                                     KMO_TRY_EXIT_IF_ERROR(
00862                                         kmo_edge_nan(data_cube_list[data_cube_counter], ifu_nr));
00863                                 }
00864 
00865                                 KMO_TRY_EXIT_IF_NULL(
00866                                     data_header_list[data_cube_counter] =
00867                                         kmo_dfs_load_sub_header(frameset, tmp_str,
00868                                                                 ifu_nr, FALSE));
00869                                 cpl_propertylist_update_string(data_header_list[data_cube_counter],
00870                                                                "ESO PRO FRNAME",
00871                                                                frame_filename);
00872                                 cpl_propertylist_update_int(data_header_list[data_cube_counter],
00873                                                             "ESO PRO IFUNR",
00874                                                             ifu_nr);
00875                                 data_cube_counter++;
00876                             }
00877 
00878                             // load noise frames
00879                             override_err_msg = TRUE;
00880                             noise_cube_list[noise_cube_counter] =
00881                                 kmo_dfs_load_cube(frameset, tmp_str, ifu_nr, TRUE);
00882                             override_err_msg = FALSE;
00883                             if (noise_cube_list[noise_cube_counter] == NULL) {
00884                                 // no noise found for this IFU
00885                                 cpl_error_reset();
00886                             } else {
00887                                 if (edge_nan) {
00888                                     KMO_TRY_EXIT_IF_ERROR(
00889                                         kmo_edge_nan(noise_cube_list[noise_cube_counter], ifu_nr));
00890                                 }
00891 
00892                                 KMO_TRY_EXIT_IF_NULL(
00893                                     noise_header_list[noise_cube_counter] =
00894                                         kmo_dfs_load_sub_header(frameset, tmp_str,
00895                                                                 ifu_nr, TRUE));
00896                                 noise_cube_counter++;
00897                             }
00898 
00899                             // check for every iteration if number of data and noise
00900                             // frames is the same
00901                             if (noise_cube_counter > 0) {
00902                                 KMO_TRY_ASSURE(data_cube_counter == noise_cube_counter,
00903                                                CPL_ERROR_ILLEGAL_INPUT,
00904                                                "Frame No. %d (%s) has no noise frame "
00905                                                "while the preceeding ones had!",
00906                                                i+1, frame_filename);
00907                             }
00908                         } // end if valid_data
00909                     } // end if (ifu_nr > 0)
00910                 }
00911 
00912                 kmo_free_fits_desc(&desc);
00913                 cpl_free(tmp_str); tmp_str = NULL;
00914             } // for i = nr_frames
00915             KMO_TRY_CHECK_ERROR_STATE();
00916 
00917             //
00918             // combine data
00919             //
00920             KMO_TRY_EXIT_IF_ERROR(
00921                 kmo_priv_combine(data_cube_list,
00922                                  noise_cube_list,
00923                                  data_header_list,
00924                                  noise_header_list,
00925                                  data_cube_counter,
00926                                  noise_cube_counter,
00927                                  name,
00928                                  ifus_txt,
00929                                  method,
00930                                  "BCS",
00931                                  fmethod,
00932                                  filename,
00933                                  cmethod,
00934                                  cpos_rej,
00935                                  cneg_rej,
00936                                  citer,
00937                                  cmin,
00938                                  cmax,
00939                                  extrapol_enum,
00940                                  flux,
00941                                  &cube_combined_data,
00942                                  &cube_combined_noise,
00943                                  &exp_mask));
00944 
00945             //
00946             // save data
00947             //
00948             /* save data and noise (if existing) */
00949             // --- load, update & save primary header ---
00950 
00951 
00952             if (!suppress_extension) {
00953                 // setup output category COMBINE + ESO PRO CATG
00954                 KMO_TRY_EXIT_IF_NULL(
00955                     main_header = kmo_dfs_load_primary_header(frameset, "0"));
00956                 KMO_TRY_EXIT_IF_NULL(
00957                     fn_combine = cpl_sprintf("%s_%s_%s",
00958                                              COMBINE,
00959                                              cpl_propertylist_get_string(main_header, CPL_DFS_PRO_CATG),
00960                                              name_vec[nv]));
00961                 KMO_TRY_EXIT_IF_NULL(
00962                     fn_mask = cpl_sprintf("%s_%s_%s",
00963                                           EXP_MASK,
00964                                           cpl_propertylist_get_string(main_header, CPL_DFS_PRO_CATG),
00965                                           name_vec[nv]));
00966                 cpl_propertylist_delete(main_header); main_header = NULL;
00967             } else {
00968                 KMO_TRY_EXIT_IF_NULL(
00969                     fn_combine = cpl_sprintf("%s_%d", COMBINE, suppress_index));
00970                 KMO_TRY_EXIT_IF_NULL(
00971                     fn_mask = cpl_sprintf("%s_%d", EXP_MASK, suppress_index++));
00972             }
00973 
00974             frame = cpl_frameset_get_first(frameset);
00975             KMO_TRY_EXIT_IF_ERROR(
00976                 kmo_dfs_save_main_header(frameset, fn_combine, "", frame, NULL, parlist, cpl_func));
00977             KMO_TRY_EXIT_IF_ERROR(
00978                 kmo_dfs_save_main_header(frameset, fn_mask, "", frame, NULL, parlist, cpl_func));
00979 
00980             if (data_header_list[0] != NULL) {
00981                 if (cpl_propertylist_has(data_header_list[0], "ESO PRO FRNAME")) {
00982                     cpl_propertylist_erase(data_header_list[0], "ESO PRO FRNAME");
00983                 }
00984                 if (cpl_propertylist_has(data_header_list[0], "ESO PRO IFUNR")) {
00985                     cpl_propertylist_erase(data_header_list[0], "ESO PRO IFUNR");
00986                 }
00987             }
00988             KMO_TRY_CHECK_ERROR_STATE();
00989 
00990             KMO_TRY_EXIT_IF_ERROR(
00991                 kmo_dfs_save_cube(cube_combined_data, fn_combine, "", data_header_list[0], 0./0.));
00992             KMO_TRY_EXIT_IF_ERROR(
00993                 kmo_dfs_save_cube(cube_combined_noise, fn_combine, "", noise_header_list[0], 0./0.));
00994             KMO_TRY_EXIT_IF_ERROR(
00995                 kmo_dfs_save_image(exp_mask, fn_mask, "", data_header_list[0], 0./0.));
00996 
00997             cpl_imagelist_delete(cube_combined_data); cube_combined_data = NULL;
00998             cpl_imagelist_delete(cube_combined_noise); cube_combined_noise = NULL;
00999             cpl_image_delete(exp_mask); exp_mask = NULL;
01000 
01001             if (data_cube_list != NULL) {
01002                 for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01003                     cpl_imagelist_delete(data_cube_list[i]); data_cube_list[i] = NULL;
01004                 }
01005             }
01006             if (noise_cube_list != NULL) {
01007                 for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01008                     cpl_imagelist_delete(noise_cube_list[i]); noise_cube_list[i] = NULL;
01009                 }
01010             }
01011             if (data_header_list != NULL) {
01012                 for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01013                     cpl_propertylist_delete(data_header_list[i]);
01014                     data_header_list[i] = NULL;
01015                 }
01016             }
01017             if (noise_header_list != NULL) {
01018                 for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01019                     cpl_propertylist_delete(noise_header_list[i]);
01020                     noise_header_list[i] = NULL;
01021                 }
01022             }
01023             cpl_free(fn_combine); fn_combine = NULL;
01024             cpl_free(fn_mask); fn_mask = NULL;
01025         }
01026     }
01027     KMO_CATCH
01028     {
01029         KMO_CATCH_MSG();
01030         ret_val = -1;
01031     }
01032 
01033     cpl_propertylist_delete(main_header); main_header = NULL;
01034     cpl_vector_delete(ifus); ifus = NULL;
01035     cpl_imagelist_delete(cube_combined_data); cube_combined_data = NULL;
01036     cpl_imagelist_delete(cube_combined_noise); cube_combined_noise = NULL;
01037     cpl_image_delete(exp_mask); exp_mask = NULL;
01038 
01039     if (data_cube_list != NULL) {
01040         for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01041             cpl_imagelist_delete(data_cube_list[i]); data_cube_list[i] = NULL;
01042         }
01043         cpl_free(data_cube_list); data_cube_list = NULL;
01044     }
01045 
01046     if (noise_cube_list != NULL) {
01047         for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01048             cpl_imagelist_delete(noise_cube_list[i]); noise_cube_list[i] = NULL;
01049         }
01050         cpl_free(noise_cube_list); noise_cube_list = NULL;
01051     }
01052 
01053     if (data_header_list != NULL) {
01054         for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01055             cpl_propertylist_delete(data_header_list[i]);
01056             data_header_list[i] = NULL;
01057         }
01058         cpl_free(data_header_list); data_header_list = NULL;
01059     }
01060 
01061     if (noise_header_list != NULL) {
01062         for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01063             cpl_propertylist_delete(noise_header_list[i]);
01064             noise_header_list[i] = NULL;
01065         }
01066         cpl_free(noise_header_list); noise_header_list = NULL;
01067     }
01068 
01069     if (name_vec != NULL) {
01070         for (i = 0; i < nr_frames*KMOS_NR_IFUS; i++) {
01071             cpl_free(name_vec[i]); name_vec[i] = NULL;
01072         }
01073         cpl_free(name_vec); name_vec = NULL;
01074     }
01075     cpl_free(mapping_mode);mapping_mode = NULL;
01076 
01077     return ret_val;
01078 }
01079