diff --git a/README.md b/README.md index 1517ad0e6c738b4810f276fcd6618f1321d47ace..eca4887cc16be6bc481399323682dfcaaae72884 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,25 @@ -# containers +# LOFAR-IT Containers +Containers source by the Italian LOFAR group. + +## Prefactor3 + +This container is based on the "lofaruser/imaging-pipeline:v3.10" base image, and it is available from Docker Hub as "lofarit/prefactor3:pipeline_v3.10". + +How to run the container with Docker: + + docker run --rm -v $INPUT_DATA_FOLDER:/input_data,$OUTPUT_DATA_FOLDER:/output_data -it lofarit/prefactor3:pipeline_v3.10 + +How to run the container wth Singularity: + + singularity run --pid --writable-tmpfs --containall --cleanenv -B$INPUT_DATA_FOLDER:/input_data,$OUTPUT_DATA_FOLDER:/output_data docker://lofarit/prefactor3:pipeline_v3.10 + +In both cases you have to set the $INPUT_DATA_FOLDER and $OUTPUT_DATA_FOLDER to the input and output data folders respectively, on the host system (i.e. the machine on which you are running the container). Note that the output data folder must exists and have write permissions (if using Singularty, by the user running the container). + +How to start prefactor3 once in the container: + + $ ./run_pipelines.sh + +This command will run the calibrator and target pipelines in sequence. Feel free to have a look and change the files as pipeline.cfg, Pre-Facet-Calibrator.parset, Pre-Facet-Target.parset and run_pipelines.sh itself in the /home/lofar directory inside the container to suit your needs. + +Remember that the contents of the container (excluding data on your external volumes as the input/output data directories) will be wiped when you exit the container. \ No newline at end of file diff --git a/prefactor3/Dockerfile b/prefactor3/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..1fc9d50cd14d9ad03bb38889f0f6c00da7eb7962 --- /dev/null +++ b/prefactor3/Dockerfile @@ -0,0 +1,71 @@ +FROM lofaruser/imaging-pipeline:v3.10 + +# Set non-interactive +ENV DEBIAN_FRONTEND noninteractive + +# Always update when extending base images +RUN apt update + + +#------------------------ +# Install deps +#------------------------ + +# Git, Curl, sudo and Nano +RUN apt-get install git curl sudo nano -y + + +#------------------------ +# Lofar user +#------------------------ + +# Add group. We chose GID 65527 to try avoiding conflicts. +RUN groupadd -g 65527 lofar + +# Add user. We chose UID 65527 to try avoiding conflicts. +RUN useradd lofar -d /home/lofar -u 65527 -g 65527 -m -s /bin/bash + +# Add metuaser user to sudoers +RUN adduser lofar sudo + +# No pass sudo (for everyone, actually) +COPY sudoers /etc/sudoers + + +#------------------------ +# Get prefactor +#------------------------ + +RUN cd /opt && git clone https://github.com/lofar-astron/prefactor +RUN cd /opt/prefactor && git pull && git checkout d4f18ff # V3.0 tag hash + +# Add conf and run scripts +COPY pipeline.cfg /home/lofar/pipeline.cfg +COPY Pre-Facet-Calibrator.parset /home/lofar/Pre-Facet-Calibrator.parset +COPY Pre-Facet-Target.parset /home/lofar/Pre-Facet-Target.parset +COPY run_pipelines.sh /home/lofar/run_pipelines.sh +COPY data/input_data /input_data +COPY data/output_data /output_data +RUN chmod 755 -R /home/lofar/run_pipelines.sh && chown lofar:lofar /home/lofar && chown -R lofar:lofar /input_data && chown -R lofar:lofar /output_data + +# This is required mainly for Singularity +RUN mv /home/lofar /home/vanilla_lofar +RUN ln -s /tmp/lofarhome /home/lofar +RUN rm -rf /tmp/lofarhome + + +#---------------------- +# Entrypoint +#---------------------- + +# Copy entrypoint +COPY entrypoint.sh / + +# Give right permissions +RUN chmod 755 /entrypoint.sh + +# Set entrypoint +ENTRYPOINT ["/entrypoint.sh"] + +# Set user lofar +USER lofar diff --git a/prefactor3/Pre-Facet-Calibrator.parset b/prefactor3/Pre-Facet-Calibrator.parset new file mode 100644 index 0000000000000000000000000000000000000000..78e1750fd1745dbf4cc0d8d842442befb1c6a48b --- /dev/null +++ b/prefactor3/Pre-Facet-Calibrator.parset @@ -0,0 +1,1179 @@ +########################################################################## +# Pre-Facet Calibrator Calibration Pipeline v3.0 (04/09/2019) # +# # +# Calibrator part of the basic Pre-Facet calibration pipeline: # +# - requires LOFAR software version >= 3.1.0 # +# - requires losoto software version >= 2.0.0 # +# - expects shared filesystem, that all nodes can reach all files! # +# (E.g. a single workstation or compute cluster with shared filesystem # +# doesn't work on multiple nodes on CEP3.) # +########################################################################## + +########################################## +### parameters you will need to adjust. ## +########################################## +## information about the calibrator data +! cal_input_path = /input_data/calib ## specify the directory where your calibrator data is stored +! cal_input_pattern = *.MS ## regular expression pattern of all your calibrator files + +## location of the software +! prefactor_directory = /opt/prefactor/ ## path to your prefactor copy +! losoto_directory = /opt/lofarsoft ## path to your local LoSoTo installation +! aoflagger = /opt/lofarsoft/bin/aoflagger ## path to your aoflagger executable + +########################################## +### parameters you may need to adjust ## +########################################## + +! refant = 'CS001HBA0' ## name of the station that will be used as a reference for the phase-plots, 'closest' will reference to the spatially closest unflagged antenna +! flag_baselines = [] ## NDPPP-compatible pattern for baselines or stations to be flagged (may be an empty list, i.e.: [] ) +! process_baselines_cal = *& ## performs A-Team-clipping/demixing and direction-independent phase-only self-calibration only on these baselines. Choose [CR]S*& if you want to process only cross-correlations and remove international stations. +! filter_baselines = {{ process_baselines_cal }} ## selects only this set of baselines to be processed. Choose [CR]S*& if you want to process only cross-correlations and remove international stations. +! do_smooth = False ## enable or disable baseline-based smoothing +! rfistrategy = HBAdefault.rfis ## strategy to be applied with the statistical flagger (AOFlagger) +! max2interpolate = 30 ## amount of channels in which interpolation should be performed for deriving the bandpass +! ampRange = [0, 0] ## range of median amplitudes accepted per station. Use [0, 0] to use typical ranges of [50, 225] for the current correlator or [0.0004, 0.0018] for the old one. +! skip_international = True ## skip fitting the bandpass for international stations (this avoids flagging them in many cases) +! raw_data = False ## use autoweight, set to True in case you are using raw data +! propagatesolutions = True ## use already derived solutions as initial guess for the upcoming time slot +! flagunconverged = False ## flag solutions for solves that did not converge (if they were also detected to diverge) +! maxStddev = -1.0 ## Maximum allowable standard deviation when outlier clipping is done. For phases, this should value should be in radians, for amplitudes in log(amp). If None (or negative), a value of 0.1 rad is used for phases and 0.01 for amplitudes + +# options for solution transfer (for non-trusted calibrator observations) +! solutions2transfer = {{ prefactor_directory }}/solutions/3C48.h5 ## location of h5parm reference solutions to transfer +! antennas2transfer = [FUSPID].* ## regular expression of antennas which solutions should be transferred to the final solution set, if none of the trusted calibrators have been observed (by default: all international stations) + +# demixing options (only used if demix step is added to the prep_cal_strategy variable) +! demix_sources = [CasA,CygA] ## choose sources to demix (provided as list) +! demix_target = "" ## if given, the target source model (its patch in the SourceDB) is taken into account when solving +! demix_freqstep = 16 ## number of channels to average when demixing. +! demix_timestep = 10 ## number of time slots to average when demixing + +# definitions for pipeline options -- do not change! +! default_flagging = flagbaseline,flagelev,flagamp ## regular flagging after pre-processing by the observatory pipelines +! raw_flagging = flagedge,aoflag,{{ default_flagging }} ## full flagging (usually only necessary for raw data) +! 1st_order = ct,plotTEC,residuals ## Do not change! Only cal_ion should be edited if needed +! 3rd_order = ct3,plotTEC,plotTEC3,residuals3 ## Do not change! Only cal_ion should be edited if needed +! demix = demix, ## Do not change! Only demix_step should be edited if needed +! full_apply = ,apply_PA,apply_bandpass,apply_clock,apply_beam,apply_FR,apply_TEC ## Do not change! +! none = ## Do not change! + +# pipeline options +! initial_flagging = {{ default_flagging }} ## choose {{ raw_flagging }} if you process raw data +! demix_step = {{ none }} ## choose {{ demix }} if you want to demix +! cal_ion = {{ 1st_order }},smooth ## choose {{ 3rd_order }} if you want to include 3rd order ionospheric effects (may be useful for LBA < 35 MHz), add smooth if you want to use the median of the clock in time (suggested for HBA+LB) +! tables2export = clock ## comma-separated list of tables to export from the ionospheric calibration step (cal_ion) +! final_apply = {{ none }} ## choose {{ full_apply }} if you want to apply all extracted solutions to the calibrator field + +########################################## +### parameters for pipeline performance ## +########################################## + +! num_proc_per_node = input.output.max_per_node ## number of processes to use per step per node (usually max_per_node from pipeline.cfg) +! num_proc_per_node_limit = 4 ## number of processes to use per step per node for tasks with high i/o (dppp or cp) or memory (eg calibration) +! max_dppp_threads = 10 ## number of threads per process for NDPPP +! memoryperc = 20 ## maximum of memory used for aoflagger in raw_flagging mode in percent +! min_length = 50 ## minimum amount of subbands to concatenate in frequency necessary to perform the wide-band flagging in the RAM. It data is too big aoflag will use indirect-read. +! overhead = 0.8 ## Only use this fraction of the available memory for deriving the amount of data to be concatenated. +! min_separation = 30 ## minimal accepted distance to an A-team source on the sky in degrees (will raise a WARNING) +! max_separation_arcmin = 1.0 ## maximum distance to the phase center for which a skymodel will be still accepted. Change with caution! + +! error_tolerance = False ## set this to True if you want the pipeline run to continue if single bands fail + +########################################## +### parameters you may want to adjust ## +########################################## + +## main directories +! lofar_directory = $LOFARROOT ## base directory of your LOFAR installation +! job_directory = input.output.job_directory ## directory of the prefactor outputs +! working_directory = input.output.working_directory/input.output.job_name ## specify the working_directory (intermediate data products) +! log_file = input.output.log_file ## location of the logfile +! mapfile_dir = input.output.mapfile_dir ## specify mapfile directory + +## script and plugin directories +! scripts = {{ prefactor_directory }}/scripts +pipeline.pluginpath = {{ prefactor_directory }}/plugins + +## skymodel directory +! calibrator_path_skymodel = {{ prefactor_directory }}/skymodels +! A-team_skymodel = {{ calibrator_path_skymodel }}/Ateam_LBA_CC.skymodel + +## result directories +! results_directory = {{ job_directory }}/results ## location of the results +! inspection_directory = {{ results_directory }}/inspection ## directory where the inspection plots will be stored +! cal_values_directory = {{ results_directory }}/cal_values ## directory where the final h5parm solution set will be stored + +## calibrator solutions +! cal_solutions = {{ cal_values_directory }}/cal_solutions.h5 + +## averaging for the calibrator data +! avg_timeresolution = 4. ## average to 4 sec/timeslot +! avg_freqresolution = 48.82kHz ## average to 48.82 kHz/ch (= 4 ch/SB) +! bandpass_freqresolution = 195.3125kHz ## resolution of the bandpass table is 195.3125kHz (= 1 ch/SB) + +######################################################## +## ## +## BEGIN PIPELINE: DO NOT UPDATE BELOW THIS LINE! ## +## ## +######################################################## + +# which steps to run +pipeline.steps = [prep, PA, FR, bandpass, ion, finalize] + + +# pipeline substeps +pipeline.steps.prep = [createmap_cal, combine_data_map, check_Ateam_separation, mk_cal_values_dir, createmap_prepcal, createmap_instcal, create_ateam_model_map, make_sourcedb_ateam, expand_sourcedb_ateam, ndppp_prep_cal, combine_data_cal_map, ms_concat, ms_concat_map, expand_memory_map, aoflag, sky_cal, make_sourcedb, expand_sourcedb, expand_skymodel, calib_cal_parmmap, h5imp_cal_map, smooth_data, predict_cal] + +pipeline.steps.PA = [calib_cal, h5imp_cal_PA, prepare_losoto_PA, process_losoto_PA, h5exp_cal_PA, apply_PA, apply_beam] +pipeline.steps.FR = [smooth_corrected, calib_cal, h5imp_cal_FR, prepare_losoto_FR, process_losoto_FR, h5exp_cal_FR, apply_FR] +pipeline.steps.bandpass = [smooth_corrected, calib_cal2, h5imp_cal_bandpass, prepare_losoto_bandpass, prepare_losoto_bandpasstrans, process_losoto_bandpass, h5exp_cal_bandpass, transfer_solutions, apply_PA, apply_bandpass, apply_beam, apply_FR] +pipeline.steps.ion = [smooth_corrected, calib_cal2, h5imp_cal_ion, prepare_losoto_ion, process_losoto_ion, h5exp_cal_ion] +pipeline.steps.finalize = [h5parm_name {{ final_apply }}, make_summary] + +############################### +## Mapping calibrator files ## +############################### + +# generate a mapfile of all the calibrator data +createmap_cal.control.kind = plugin +createmap_cal.control.type = createMapfile +createmap_cal.control.method = mapfile_from_folder +createmap_cal.control.mapfile_dir = {{ mapfile_dir }} +createmap_cal.control.filename = createmap_cal.mapfile +createmap_cal.control.folder = {{ cal_input_path }} +createmap_cal.control.pattern = {{ cal_input_pattern }} + +# combine all entries into one mapfile, for the sortmap script +combine_data_map.control.kind = plugin +combine_data_map.control.type = createMapfile +combine_data_map.control.method = mapfile_all_to_one +combine_data_map.control.mapfile_dir = {{ mapfile_dir }} +combine_data_map.control.filename = combine_data_map.mapfile +combine_data_map.control.mapfile_in = createmap_cal.output.mapfile + +# warn for potential nearby A-Team sources +check_Ateam_separation.control.type = pythonplugin +check_Ateam_separation.control.executable = {{ scripts }}/check_Ateam_separation.py +check_Ateam_separation.control.mapfile_in = combine_data_map.output.mapfile +check_Ateam_separation.control.inputkey = MSfile +check_Ateam_separation.argument.min_separation = {{ min_separation }} +check_Ateam_separation.argument.outputimage = {{ inspection_directory }}/A-Team_elevation_calibrator.png +check_Ateam_separation.argument.flags = [MSfile] + +############################# +## Prepare for demixing ## +############################# +# generate a mapfile of the calibrator +createmap_prepcal.control.kind = plugin +createmap_prepcal.control.type = makeResultsMapfile +createmap_prepcal.control.mapfile_dir = {{ mapfile_dir }} +createmap_prepcal.control.filename = createmap_prepcal.mapfile +createmap_prepcal.control.mapfile_in = createmap_cal.output.mapfile +createmap_prepcal.control.target_dir = {{ job_directory }} +createmap_prepcal.control.make_target_dir = False +createmap_prepcal.control.new_suffix = .ndppp_prep_cal + +# generate a mapfile for the instrument table of the calibrator +createmap_instcal.control.kind = plugin +createmap_instcal.control.type = changeMapfile +createmap_instcal.control.mapfile_in = createmap_prepcal.output.mapfile +createmap_instcal.control.join_files = instrument +createmap_instcal.control.newname = createmap_instcal.mapfile + +# create a mapfile with the A-Team skymodel, length = 1 +create_ateam_model_map.control.kind = plugin +create_ateam_model_map.control.type = addListMapfile +create_ateam_model_map.control.hosts = ['localhost'] +create_ateam_model_map.control.files = [ {{ A-team_skymodel }} ] +create_ateam_model_map.control.mapfile_dir = {{ mapfile_dir }} +create_ateam_model_map.control.filename = ateam_model_name.mapfile + +# make sourcedbs from the A-Team skymodel, length = 1 +make_sourcedb_ateam.control.kind = recipe +make_sourcedb_ateam.control.type = executable_args +make_sourcedb_ateam.control.executable = {{ lofar_directory }}/bin/makesourcedb +make_sourcedb_ateam.control.error_tolerance = {{ error_tolerance }} +make_sourcedb_ateam.control.args_format = lofar +make_sourcedb_ateam.control.outputkey = out +make_sourcedb_ateam.control.mapfile_in = create_ateam_model_map.output.mapfile +make_sourcedb_ateam.control.inputkey = in +make_sourcedb_ateam.argument.format = < +make_sourcedb_ateam.argument.outtype = blob + +# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles +expand_sourcedb_ateam.control.kind = plugin +expand_sourcedb_ateam.control.type = expandMapfile +expand_sourcedb_ateam.control.mapfile_in = make_sourcedb_ateam.output.mapfile +expand_sourcedb_ateam.control.mapfile_to_match = createmap_cal.output.mapfile +expand_sourcedb_ateam.control.mapfile_dir = {{ mapfile_dir }} +expand_sourcedb_ateam.control.filename = expand_sourcedb_ateam.datamap + + +############################### +## Prepare calibrator ## +############################### +# create the cal_values_directory if needed +mk_cal_values_dir.control.kind = plugin +mk_cal_values_dir.control.type = makeDirectory +mk_cal_values_dir.control.directory = {{ cal_values_directory }} + +# run NDPPP on the calibrator data +ndppp_prep_cal.control.type = dppp +ndppp_prep_cal.control.max_per_node = {{ num_proc_per_node_limit }} +ndppp_prep_cal.control.error_tolerance = {{ error_tolerance }} +ndppp_prep_cal.argument.numthreads = {{ max_dppp_threads }} +ndppp_prep_cal.argument.msin = createmap_cal.output.mapfile # The input data. +ndppp_prep_cal.argument.msin.datacolumn = DATA +ndppp_prep_cal.argument.msin.baseline = {{ filter_baselines }} +ndppp_prep_cal.argument.msin.autoweight = {{ raw_data }} +ndppp_prep_cal.argument.msout.datacolumn = DATA +ndppp_prep_cal.argument.msout.writefullresflag = False +ndppp_prep_cal.argument.msout.overwrite = True +ndppp_prep_cal.argument.msout.storagemanager = "Dysco" +ndppp_prep_cal.argument.msout.storagemanager.databitrate = 0 +ndppp_prep_cal.argument.steps = [{{ initial_flagging }},{{ demix_step }}avg] +ndppp_prep_cal.argument.flagedge.type = preflagger +ndppp_prep_cal.argument.flagedge.chan = [0..nchan/32-1,31*nchan/32..nchan-1] # we are running on a single subband +ndppp_prep_cal.argument.aoflag.type = aoflagger +ndppp_prep_cal.argument.aoflag.memoryperc = {{ memoryperc }} +ndppp_prep_cal.argument.aoflag.keepstatistics = false +ndppp_prep_cal.argument.flagbaseline.type = preflagger +ndppp_prep_cal.argument.flagbaseline.baseline = {{ flag_baselines }} +ndppp_prep_cal.argument.flagelev.type = preflagger +ndppp_prep_cal.argument.flagelev.elevation = 0deg..20deg +ndppp_prep_cal.argument.flagamp.type = preflagger +ndppp_prep_cal.argument.flagamp.amplmin = 1e-30 +ndppp_prep_cal.argument.avg.type = average +ndppp_prep_cal.argument.avg.timeresolution = {{ avg_timeresolution }} +ndppp_prep_cal.argument.avg.freqresolution = {{ avg_freqresolution }} +ndppp_prep_cal.argument.flagamp.type = preflagger +ndppp_prep_cal.argument.flagamp.amplmin = 1e-30 +ndppp_prep_cal.argument.demix.type = demixer +ndppp_prep_cal.argument.demix.baseline = {{ process_baselines_cal }} +ndppp_prep_cal.argument.demix.demixfreqstep = {{ demix_freqstep }} +ndppp_prep_cal.argument.demix.demixtimestep = {{ demix_timestep }} +ndppp_prep_cal.argument.demix.ignoretarget = False +ndppp_prep_cal.argument.demix.targetsource = {{ demix_target }} +ndppp_prep_cal.argument.demix.subtractsources = {{ demix_sources }} +ndppp_prep_cal.argument.demix.ntimechunk = {{ max_dppp_threads }} +ndppp_prep_cal.argument.demix.skymodel = expand_sourcedb_ateam.output.mapfile +ndppp_prep_cal.argument.demix.freqstep = 1 +ndppp_prep_cal.argument.demix.timestep = 1 +ndppp_prep_cal.argument.demix.instrumentmodel = createmap_instcal.output.mapfile + +# combine all entries into one mapfile (just for the find_skymodel_cal_auto script) +combine_data_cal_map.control.kind = plugin +combine_data_cal_map.control.type = createMapfile +combine_data_cal_map.control.method = mapfile_all_to_one +combine_data_cal_map.control.mapfile_dir = {{ mapfile_dir }} +combine_data_cal_map.control.filename = combine_data_cal_map.mapfile +combine_data_cal_map.control.mapfile_in = ndppp_prep_cal.output.mapfile + +# virtually concatenate calibrator subbands +ms_concat.control.type = pythonplugin +ms_concat.control.executable = {{ scripts }}/concat_MS.py +ms_concat.control.error_tolerance = {{ error_tolerance }} +ms_concat.argument.filename = concatmapfile.mapfile +ms_concat.argument.mapfile_dir = {{ mapfile_dir }} +ms_concat.argument.min_length = {{ min_length }} +ms_concat.argument.overhead = {{ overhead }} +ms_concat.argument.flags = [combine_data_cal_map.output.mapfile,outputkey] + +# convert the output of ms_concat into usable mapfiles +ms_concat_map.control.kind = plugin +ms_concat_map.control.type = mapfilenamesFromMapfiles +ms_concat_map.control.mapfile_concatmap = ms_concat.output.concatmapfile.mapfile + +# convert the output of ms_concat_target into usable mapfiles +expand_memory_map.control.kind = plugin +expand_memory_map.control.type = expandMapfile +expand_memory_map.control.mapfile_in = ms_concat.output.memory.mapfile +expand_memory_map.control.mapfile_to_match = ms_concat_map.output.concatmap +expand_memory_map.control.mapfile_dir = {{ mapfile_dir }} +expand_memory_map.control.filename = expand_memory_map.mapfile + +# run aoflagger on the concatenated data +aoflag.control.kind = recipe +aoflag.control.type = executable_args +aoflag.control.inplace = True +aoflag.control.executable = {{ aoflagger }} +aoflag.control.max_per_node = 1 +aoflag.control.error_tolerance = {{ error_tolerance }} +aoflag.control.mapfiles_in = [ms_concat_map.output.concatmap,expand_memory_map.output.mapfile] +aoflag.control.inputkeys = [msin,memory] +aoflag.control.args_format = wsclean +aoflag.argument.strategy = {{ prefactor_directory }}/rfistrategies/{{ rfistrategy }} +aoflag.argument.flags = [-v,memory,-combine-spws,msin] + +# find automatically the calibrator sky model +sky_cal.control.type = pythonplugin +sky_cal.control.executable = {{ scripts }}/find_skymodel_cal.py +sky_cal.control.error_tolerance = {{ error_tolerance }} +sky_cal.argument.flags = [combine_data_cal_map.output.mapfile] +sky_cal.argument.DirSkymodelCal = {{ calibrator_path_skymodel }} +sky_cal.argument.max_separation_arcmin = {{ max_separation_arcmin }} + +# make the sourcedb +make_sourcedb.control.kind = recipe +make_sourcedb.control.type = executable_args +make_sourcedb.control.executable = {{ lofar_directory }}/bin/makesourcedb +make_sourcedb.control.error_tolerance = {{ error_tolerance }} +make_sourcedb.control.args_format = lofar +make_sourcedb.control.outputkey = out +make_sourcedb.control.mapfile_in = sky_cal.output.SkymodelCal.mapfile +make_sourcedb.control.inputkey = in +make_sourcedb.argument.format = < +make_sourcedb.argument.outtype = blob + +# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles +expand_sourcedb.control.kind = plugin +expand_sourcedb.control.type = expandMapfile +expand_sourcedb.control.mapfile_in = make_sourcedb.output.mapfile +expand_sourcedb.control.mapfile_to_match = ndppp_prep_cal.output.mapfile +expand_sourcedb.control.mapfile_dir = {{ mapfile_dir }} +expand_sourcedb.control.filename = expand_sourcedb.mapfile + +# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles +expand_skymodel.control.kind = plugin +expand_skymodel.control.type = expandMapfile +expand_skymodel.control.mapfile_in = sky_cal.output.SkymodelName.mapfile +expand_skymodel.control.mapfile_to_match = ndppp_prep_cal.output.mapfile +expand_skymodel.control.mapfile_dir = {{ mapfile_dir }} +expand_skymodel.control.filename = expand_skymodel.mapfile + +# generate mapfile with the h5parm names to be used in the calib_cal steps +calib_cal_parmmap.control.kind = plugin +calib_cal_parmmap.control.type = createMapfile +calib_cal_parmmap.control.method = add_suffix_to_file +calib_cal_parmmap.control.mapfile_in = ndppp_prep_cal.output.mapfile +calib_cal_parmmap.control.add_suffix_to_file = /instrument.h5 +calib_cal_parmmap.control.mapfile_dir = {{ mapfile_dir }} +calib_cal_parmmap.control.filename = calib_cal_h5parms.mapfile + +# generate a mapfile with all files in a single entry +h5imp_cal_map.control.kind = plugin +h5imp_cal_map.control.type = compressMapfile +h5imp_cal_map.control.mapfile_in = calib_cal_parmmap.output.mapfile +h5imp_cal_map.control.mapfile_dir = {{ mapfile_dir }} +h5imp_cal_map.control.filename = h5imp_cal_map.mapfile + +# baseline-dependent smoothing +smooth_data.control.type = executable_args +smooth_data.control.inplace = True +smooth_data.control.max_per_node = {{ num_proc_per_node }} +smooth_data.control.error_tolerance = {{ error_tolerance }} +smooth_data.control.executable = {{ scripts }}/BLsmooth.py +smooth_data.control.mapfile_in = ndppp_prep_cal.output.mapfile +smooth_data.control.inputkey = msin +smooth_data.argument.flags = [-S,{{ do_smooth }},-r,-i,DATA,-o,SMOOTHED_DATA,msin] + +# baseline-dependent smoothing +smooth_corrected.control.type = executable_args +smooth_corrected.control.inplace = True +smooth_corrected.control.max_per_node = {{ num_proc_per_node }} +smooth_corrected.control.error_tolerance = {{ error_tolerance }} +smooth_corrected.control.executable = {{ scripts }}/BLsmooth.py +smooth_corrected.control.mapfile_in = ndppp_prep_cal.output.mapfile +smooth_corrected.control.inputkey = msin +smooth_corrected.argument.flags = [-S,{{ do_smooth }},-r,-i,CORRECTED_DATA,-o,SMOOTHED_DATA,msin] + +# predict to save time +predict_cal.control.type = dppp +predict_cal.control.inplace = True +predict_cal.control.max_per_node = {{ num_proc_per_node_limit }} +predict_cal.control.error_tolerance = {{ error_tolerance }} +predict_cal.control.mapfiles_in = [ndppp_prep_cal.output.mapfile,expand_sourcedb.output.mapfile,expand_skymodel.output.mapfile] +predict_cal.control.inputkeys = [msfile,sourcedb,skymodel] +predict_cal.argument.msin = msfile +predict_cal.argument.numthreads = {{ max_dppp_threads }} +predict_cal.argument.msin.datacolumn = SMOOTHED_DATA +predict_cal.argument.msout.datacolumn = MODEL_DATA +predict_cal.argument.msout.storagemanager = "Dysco" +predict_cal.argument.msout.storagemanager.databitrate = 0 +predict_cal.argument.steps = [predict] +predict_cal.argument.predict.type = predict +predict_cal.argument.predict.sourcedb = sourcedb +predict_cal.argument.predict.sources = skymodel +predict_cal.argument.predict.usebeammodel = False # TODO: put to true for large sky +predict_cal.argument.predict.usechannelfreq = False +predict_cal.argument.predict.beammode = array_factor + +# now run NDPPP on the averaged calibrator data (rotation+diagonal) +calib_cal.control.type = dppp +calib_cal.control.inplace = True +calib_cal.control.max_per_node = {{ num_proc_per_node_limit }} +calib_cal.control.error_tolerance = {{ error_tolerance }} +calib_cal.control.mapfiles_in = [ndppp_prep_cal.output.mapfile,calib_cal_parmmap.output.mapfile] +calib_cal.control.inputkeys = [msfile,h5parm] +calib_cal.argument.msin = msfile +calib_cal.argument.numthreads = {{ max_dppp_threads }} +calib_cal.argument.msin.datacolumn = SMOOTHED_DATA +calib_cal.argument.steps = [solve] +calib_cal.argument.solve.type = ddecal +calib_cal.argument.solve.mode = rotation+diagonal +calib_cal.argument.solve.h5parm = h5parm +calib_cal.argument.solve.usemodelcolumn = True +calib_cal.argument.solve.uvlambdamin = 300 +calib_cal.argument.solve.maxiter = 50 +calib_cal.argument.solve.nchan = 1 +calib_cal.argument.solve.solint = 1 +calib_cal.argument.solve.propagatesolutions = {{ propagatesolutions }} +calib_cal.argument.solve.propagateconvergedonly = True +calib_cal.argument.solve.flagunconverged = {{ flagunconverged }} +calib_cal.argument.solve.flagdivergedonly = True +calib_cal.argument.solve.tolerance = 1e-3 + +# now run NDPPP on the averaged calibrator data (diagonal only) +calib_cal2.control.type = dppp +calib_cal2.control.inplace = True +calib_cal2.control.max_per_node = {{ num_proc_per_node_limit }} +calib_cal2.control.error_tolerance = {{ error_tolerance }} +calib_cal2.control.mapfiles_in = [ndppp_prep_cal.output.mapfile,calib_cal_parmmap.output.mapfile] +calib_cal2.control.inputkeys = [msfile,h5parm] +calib_cal2.argument.msin = msfile +calib_cal2.argument.numthreads = {{ max_dppp_threads }} +calib_cal2.argument.msin.datacolumn = SMOOTHED_DATA +calib_cal2.argument.steps = [solve] +calib_cal2.argument.solve.type = ddecal +calib_cal2.argument.solve.mode = diagonal +calib_cal2.argument.solve.h5parm = h5parm +calib_cal2.argument.solve.usemodelcolumn = True +calib_cal2.argument.solve.uvlambdamin = 300 +calib_cal2.argument.solve.maxiter = 50 +calib_cal2.argument.solve.nchan = 1 +calib_cal2.argument.solve.solint = 1 +calib_cal2.argument.solve.propagatesolutions = {{ propagatesolutions }} +calib_cal2.argument.solve.propagateconvergedonly = True +calib_cal2.argument.solve.flagunconverged = {{ flagunconverged }} +calib_cal2.argument.solve.flagdivergedonly = True +calib_cal2.argument.solve.tolerance = 1e-3 + + +########################### +## calibrate Pol. Align ## +########################### +# collect all instrument tables into one h5parm +h5imp_cal_PA.control.kind = recipe +h5imp_cal_PA.control.type = executable_args +h5imp_cal_PA.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py +h5imp_cal_PA.control.error_tolerance = {{ error_tolerance }} +h5imp_cal_PA.control.mapfile_in = h5imp_cal_map.output.mapfile +h5imp_cal_PA.control.inputkey = h5parm +h5imp_cal_PA.control.outputkey = outh5parm +h5imp_cal_PA.argument.flags = [-q,-v,-c,h5parm] +h5imp_cal_PA.argument.outh5parm = outh5parm + +# create losoto v2 parset file +prepare_losoto_PA.control.kind = plugin +prepare_losoto_PA.control.type = makeLosotoParset +prepare_losoto_PA.control.steps = [plotP3,plotPd,plotRot3,plotA3,bkp,align,plotAlign,residual,plotPr,plotPr2] +prepare_losoto_PA.control.filename = {{ job_directory }}/losoto.parset +prepare_losoto_PA.control.global.ncpu = {{ num_proc_per_node }} +prepare_losoto_PA.control.plotP3.operation = PLOT +prepare_losoto_PA.control.plotP3.soltab = sol000/phase000 +prepare_losoto_PA.control.plotP3.axesInPlot = [time,freq] +prepare_losoto_PA.control.plotP3.axisInTable = ant +prepare_losoto_PA.control.plotP3.plotFlag = True +prepare_losoto_PA.control.plotP3.prefix = {{ inspection_directory }}/polalign_ph_ +prepare_losoto_PA.control.plotP3.refAnt = {{ refant }} +prepare_losoto_PA.control.plotP3.minmax = [-3.14,3.14] +prepare_losoto_PA.control.plotPd.operation = PLOT +prepare_losoto_PA.control.plotPd.soltab = sol000/phase000 +prepare_losoto_PA.control.plotPd.axesInPlot = [time,freq] +prepare_losoto_PA.control.plotPd.axisInTable = ant +prepare_losoto_PA.control.plotPd.axisDiff = pol +prepare_losoto_PA.control.plotPd.plotFlag = True +prepare_losoto_PA.control.plotPd.prefix = {{ inspection_directory }}/polalign_ph_poldif +prepare_losoto_PA.control.plotPd.refAnt = {{ refant }} +prepare_losoto_PA.control.plotPd.minmax = [-3.14,3.14] +prepare_losoto_PA.control.plotRot3.operation = PLOT +prepare_losoto_PA.control.plotRot3.soltab = sol000/rotation000 +prepare_losoto_PA.control.plotRot3.axesInPlot = [time,freq] +prepare_losoto_PA.control.plotRot3.axisInTable = ant +prepare_losoto_PA.control.plotRot3.plotFlag = True +prepare_losoto_PA.control.plotRot3.prefix = {{ inspection_directory }}/polalign_rotangle +prepare_losoto_PA.control.plotRot3.refAnt = {{ refant }} +prepare_losoto_PA.control.plotA3.operation = PLOT +prepare_losoto_PA.control.plotA3.soltab = sol000/amplitude000 +prepare_losoto_PA.control.plotA3.axesInPlot = [time,freq] +prepare_losoto_PA.control.plotA3.axisInTable = ant +prepare_losoto_PA.control.plotA3.plotFlag = True +prepare_losoto_PA.control.plotA3.prefix = {{ inspection_directory }}/polalign_amp_ +prepare_losoto_PA.control.bkp.operation = DUPLICATE +prepare_losoto_PA.control.bkp.soltab = sol000/phase000 +prepare_losoto_PA.control.bkp.soltabOut = phaseOrig +prepare_losoto_PA.control.align.soltab = sol000/phase000 +prepare_losoto_PA.control.align.operation = POLALIGN +prepare_losoto_PA.control.align.soltabOut = polalign +prepare_losoto_PA.control.align.average = True +prepare_losoto_PA.control.align.replace = True +prepare_losoto_PA.control.align.fitOffset = False +prepare_losoto_PA.control.align.refAnt = {{ refant }} +prepare_losoto_PA.control.plotAlign.operation = PLOT +prepare_losoto_PA.control.plotAlign.soltab = sol000/polalign +prepare_losoto_PA.control.plotAlign.axesInPlot = [time,freq] +prepare_losoto_PA.control.plotAlign.axisInTable = ant +prepare_losoto_PA.control.plotAlign.axisDiff = pol +prepare_losoto_PA.control.plotAlign.plotFlag = True +prepare_losoto_PA.control.plotAlign.prefix = {{ inspection_directory }}/polalign +prepare_losoto_PA.control.plotAlign.refAnt = {{ refant }} +prepare_losoto_PA.control.plotAlign.minmax = [-3.14,3.14] +prepare_losoto_PA.control.residual.operation = RESIDUALS +prepare_losoto_PA.control.residual.soltab = sol000/phase000 +prepare_losoto_PA.control.residual.soltabsToSub = polalign +prepare_losoto_PA.control.plotPr.operation = PLOT +prepare_losoto_PA.control.plotPr.soltab = sol000/phase000 +prepare_losoto_PA.control.plotPr.axesInPlot = [time,freq] +prepare_losoto_PA.control.plotPr.axisInTable = ant +prepare_losoto_PA.control.plotPr.axisDiff = pol +prepare_losoto_PA.control.plotPr.plotFlag = True +prepare_losoto_PA.control.plotPr.prefix = {{ inspection_directory }}/polalign_ph-res_poldif +prepare_losoto_PA.control.plotPr.refAnt = {{ refant }} +prepare_losoto_PA.control.plotPr.minmax = [-3.14,3.14] +prepare_losoto_PA.control.plotPr2.operation = PLOT +prepare_losoto_PA.control.plotPr2.soltab = sol000/phase000 +prepare_losoto_PA.control.plotPr2.axesInPlot = [time,freq] +prepare_losoto_PA.control.plotPr2.axisInTable = ant +prepare_losoto_PA.control.plotPr2.axisInCol = pol +prepare_losoto_PA.control.plotPr2.plotFlag = True +prepare_losoto_PA.control.plotPr2.prefix = {{ inspection_directory }}/polalign_ph-res_ +prepare_losoto_PA.control.plotPr2.refAnt = {{ refant }} +prepare_losoto_PA.control.plotPr2.minmax = [-3.14,3.14] + +# do the processing on the LoSoTo file +process_losoto_PA.control.kind = recipe +process_losoto_PA.control.type = executable_args +process_losoto_PA.control.inplace = True +process_losoto_PA.control.executable = {{ losoto_directory }}/bin/losoto +process_losoto_PA.control.max_per_node = {{ num_proc_per_node }} +process_losoto_PA.control.mapfile_in = h5imp_cal_PA.output.mapfile +process_losoto_PA.control.inputkey = h5in +process_losoto_PA.argument.flags = [-v,h5in,{{ job_directory }}/losoto.parset] + +# output the final soltab into an external h5parm +h5exp_cal_PA.control.kind = recipe +h5exp_cal_PA.control.type = executable_args +h5exp_cal_PA.control.inplace = True +h5exp_cal_PA.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py +h5exp_cal_PA.control.error_tolerance = {{ error_tolerance }} +h5exp_cal_PA.control.mapfile_in = h5imp_cal_PA.output.mapfile +h5exp_cal_PA.control.inputkey = h5in +h5exp_cal_PA.argument.flags = [-q,-v,h5in] +h5exp_cal_PA.argument.insoltab = polalign +h5exp_cal_PA.argument.outh5parm = {{ cal_solutions }} +h5exp_cal_PA.argument.outsolset = calibrator + +################################ +## calibrate Farady Rotation ## +################################ +# collect all instrument tables into one h5parm +h5imp_cal_FR.control.kind = recipe +h5imp_cal_FR.control.type = executable_args +h5imp_cal_FR.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py +h5imp_cal_FR.control.error_tolerance = {{ error_tolerance }} +h5imp_cal_FR.control.mapfile_in = h5imp_cal_map.output.mapfile +h5imp_cal_FR.control.inputkey = h5parm +h5imp_cal_FR.control.outputkey = outh5parm +h5imp_cal_FR.argument.flags = [-q,-v,-c,h5parm] +h5imp_cal_FR.argument.outh5parm = outh5parm + +# create losoto v2 parset file +prepare_losoto_FR.control.kind = plugin +prepare_losoto_FR.control.type = makeLosotoParset +prepare_losoto_FR.control.steps = [plotP3,plotPd,plotRot3,plotA3,faraday,plotFR,residual,plotPr,plotPr2] +prepare_losoto_FR.control.filename = {{ job_directory }}/losoto.parset +prepare_losoto_FR.control.global.ncpu = {{ num_proc_per_node }} +prepare_losoto_FR.control.plotP3.operation = PLOT +prepare_losoto_FR.control.plotP3.soltab = sol000/phase000 +prepare_losoto_FR.control.plotP3.axesInPlot = [time,freq] +prepare_losoto_FR.control.plotP3.axisInTable = ant +prepare_losoto_FR.control.plotP3.plotFlag = True +prepare_losoto_FR.control.plotP3.prefix = {{ inspection_directory }}/fr_ph_ +prepare_losoto_FR.control.plotP3.refAnt = {{ refant }} +prepare_losoto_FR.control.plotP3.minmax = [-3.14,3.14] +prepare_losoto_FR.control.plotPd.operation = PLOT +prepare_losoto_FR.control.plotPd.soltab = sol000/phase000 +prepare_losoto_FR.control.plotPd.axesInPlot = [time,freq] +prepare_losoto_FR.control.plotPd.axisInTable = ant +prepare_losoto_FR.control.plotPd.axisDiff = pol +prepare_losoto_FR.control.plotPd.plotFlag = True +prepare_losoto_FR.control.plotPd.prefix = {{ inspection_directory }}/fr_ph_poldif +prepare_losoto_FR.control.plotPd.refAnt = {{ refant }} +prepare_losoto_FR.control.plotPd.minmax = [-3.14,3.14] +prepare_losoto_FR.control.plotRot3.operation = PLOT +prepare_losoto_FR.control.plotRot3.soltab = sol000/rotation000 +prepare_losoto_FR.control.plotRot3.axesInPlot = [time,freq] +prepare_losoto_FR.control.plotRot3.axisInTable = ant +prepare_losoto_FR.control.plotRot3.plotFlag = True +prepare_losoto_FR.control.plotRot3.prefix = {{ inspection_directory }}/fr_rotangle +prepare_losoto_FR.control.plotRot3.refAnt = {{ refant }} +prepare_losoto_FR.control.plotA3.operation = PLOT +prepare_losoto_FR.control.plotA3.soltab = sol000/amplitude000 +prepare_losoto_FR.control.plotA3.axesInPlot = [time,freq] +prepare_losoto_FR.control.plotA3.axisInTable = ant +prepare_losoto_FR.control.plotA3.plotFlag = True +prepare_losoto_FR.control.plotA3.prefix = {{ inspection_directory }}/fr_amp_ +prepare_losoto_FR.control.faraday.operation = FARADAY +prepare_losoto_FR.control.faraday.soltab = sol000/rotation000 +prepare_losoto_FR.control.faraday.soltabOut = faraday +prepare_losoto_FR.control.faraday.refAnt = {{ refant }} +prepare_losoto_FR.control.faraday.maxResidual = 1. +prepare_losoto_FR.control.plotFR.operation = PLOT +prepare_losoto_FR.control.plotFR.soltab = sol000/faraday +prepare_losoto_FR.control.plotFR.axesInPlot = [time] +prepare_losoto_FR.control.plotFR.axisInTable = ant +prepare_losoto_FR.control.plotFR.prefix = {{ inspection_directory }}/fr +prepare_losoto_FR.control.residual.operation = RESIDUALS +prepare_losoto_FR.control.residual.soltab = sol000/phase000 +prepare_losoto_FR.control.residual.soltabsToSub = faraday +prepare_losoto_FR.control.plotPr.operation = PLOT +prepare_losoto_FR.control.plotPr.soltab = sol000/phase000 +prepare_losoto_FR.control.plotPr.axesInPlot = [time,freq] +prepare_losoto_FR.control.plotPr.axisInTable = ant +prepare_losoto_FR.control.plotPr.axisDiff = pol +prepare_losoto_FR.control.plotPr.plotFlag = True +prepare_losoto_FR.control.plotPr.prefix = {{ inspection_directory }}/fr_ph-res_poldif +prepare_losoto_FR.control.plotPr.refAnt = {{ refant }} +prepare_losoto_FR.control.plotPr.minmax = [-3.14,3.14] +prepare_losoto_FR.control.plotPr2.operation = PLOT +prepare_losoto_FR.control.plotPr2.soltab = sol000/phase000 +prepare_losoto_FR.control.plotPr2.axesInPlot = [time,freq] +prepare_losoto_FR.control.plotPr2.axisInTable = ant +prepare_losoto_FR.control.plotPr2.axisInCol = pol +prepare_losoto_FR.control.plotPr2.plotFlag = True +prepare_losoto_FR.control.plotPr2.prefix = {{ inspection_directory }}/fr_ph-res_ +prepare_losoto_FR.control.plotPr2.refAnt = {{ refant }} +prepare_losoto_FR.control.plotPr2.minmax = [-3.14,3.14] + +# do the processing on the LoSoTo file +process_losoto_FR.control.kind = recipe +process_losoto_FR.control.type = executable_args +process_losoto_FR.control.inplace = True +process_losoto_FR.control.executable = {{ losoto_directory }}/bin/losoto +process_losoto_FR.control.max_per_node = {{ num_proc_per_node }} +process_losoto_FR.control.mapfile_in = h5imp_cal_FR.output.mapfile +process_losoto_FR.control.inputkey = h5in +process_losoto_FR.argument.flags = [-v,h5in,{{ job_directory }}/losoto.parset] + +# output the final soltab into an external h5parm +h5exp_cal_FR.control.kind = recipe +h5exp_cal_FR.control.type = executable_args +h5exp_cal_FR.control.inplace = True +h5exp_cal_FR.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py +h5exp_cal_FR.control.error_tolerance = {{ error_tolerance }} +h5exp_cal_FR.control.mapfile_in = h5imp_cal_FR.output.mapfile +h5exp_cal_FR.control.inputkey = h5in +h5exp_cal_FR.argument.flags = [-q,-v,h5in] +h5exp_cal_FR.argument.insoltab = faraday +h5exp_cal_FR.argument.outh5parm = {{ cal_solutions }} +h5exp_cal_FR.argument.outsolset = calibrator + +################################ +## calibrate bandpass ## +################################ +# collect all instrument tables into one h5parm +h5imp_cal_bandpass.control.kind = recipe +h5imp_cal_bandpass.control.type = executable_args +h5imp_cal_bandpass.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py +h5imp_cal_bandpass.control.error_tolerance = {{ error_tolerance }} +h5imp_cal_bandpass.control.mapfile_in = h5imp_cal_map.output.mapfile +h5imp_cal_bandpass.control.inputkey = h5parm +h5imp_cal_bandpass.control.outputkey = outh5parm +h5imp_cal_bandpass.argument.flags = [-q,-v,-c,h5parm] +h5imp_cal_bandpass.argument.outh5parm = outh5parm + +# create losoto v2 parset file +prepare_losoto_bandpass.control.kind = plugin +prepare_losoto_bandpass.control.type = makeLosotoParset +prepare_losoto_bandpass.control.steps = [duplicateAbkp,plotA1,flag,flagbp,flagextend,plotA2,merge,smooth,bandpass,interp,smoothb,plotB1,plotB2,plotB3] +prepare_losoto_bandpass.control.filename = {{ job_directory }}/losoto.parset +prepare_losoto_bandpass.control.global.ncpu = {{ num_proc_per_node }} +prepare_losoto_bandpass.control.duplicateAbkp.operation = DUPLICATE +prepare_losoto_bandpass.control.duplicateAbkp.soltab = sol000/amplitude000 +prepare_losoto_bandpass.control.duplicateAbkp.soltabOut = amplitudeOrig000 +prepare_losoto_bandpass.control.plotA1.operation = PLOT +prepare_losoto_bandpass.control.plotA1.soltab = sol000/amplitude000 +prepare_losoto_bandpass.control.plotA1.axesInPlot = [time,freq] +prepare_losoto_bandpass.control.plotA1.axisInTable = ant +prepare_losoto_bandpass.control.plotA1.plotFlag = True +prepare_losoto_bandpass.control.plotA1.prefix = {{ inspection_directory }}/ampBFlag_ +prepare_losoto_bandpass.control.flag.operation = FLAG +prepare_losoto_bandpass.control.flag.soltab = sol000/amplitude000 +prepare_losoto_bandpass.control.flag.axesToFlag = [time,freq] +prepare_losoto_bandpass.control.flag.order = [100,40] +prepare_losoto_bandpass.control.flag.maxCycles = 1 +prepare_losoto_bandpass.control.flag.maxRms = 5 +prepare_losoto_bandpass.control.flag.replace = False +prepare_losoto_bandpass.control.flag.preFlagZeros = False +prepare_losoto_bandpass.control.flag.mode = smooth +prepare_losoto_bandpass.control.flagbp.operation = FLAGSTATION +prepare_losoto_bandpass.control.flagbp.soltab = sol000/amplitude000 +prepare_losoto_bandpass.control.flagbp.mode = bandpass +prepare_losoto_bandpass.control.flagbp.ampRange = {{ ampRange }} +prepare_losoto_bandpass.control.flagbp.skipInternational = {{ skip_international }} +prepare_losoto_bandpass.control.flagextend.operation = FLAGEXTEND +prepare_losoto_bandpass.control.flagextend.soltab = sol000/amplitude000 +prepare_losoto_bandpass.control.flagextend.axesToExt = [time,freq] +prepare_losoto_bandpass.control.flagextend.size = [200,80] +prepare_losoto_bandpass.control.flagextend.percent = 50 +prepare_losoto_bandpass.control.flagextend.maxCycles = 2 +prepare_losoto_bandpass.control.plotA2.operation = PLOT +prepare_losoto_bandpass.control.plotA2.soltab = sol000/amplitude000 +prepare_losoto_bandpass.control.plotA2.axesInPlot = [time,freq] +prepare_losoto_bandpass.control.plotA2.axisInTable = ant +prepare_losoto_bandpass.control.plotA2.plotFlag = True +prepare_losoto_bandpass.control.plotA2.prefix = {{ inspection_directory }}/ampAFlag_ +prepare_losoto_bandpass.control.merge.operation = REWEIGHT +prepare_losoto_bandpass.control.merge.mode = copy +prepare_losoto_bandpass.control.merge.soltab = sol000/phase000 +prepare_losoto_bandpass.control.merge.soltabImport = amplitude000 +prepare_losoto_bandpass.control.smooth.operation = SMOOTH +prepare_losoto_bandpass.control.smooth.soltab = sol000/amplitude000 +prepare_losoto_bandpass.control.smooth.pol = XX,YY +prepare_losoto_bandpass.control.smooth.axesToSmooth = [time] +prepare_losoto_bandpass.control.smooth.mode = median +prepare_losoto_bandpass.control.smooth.log = True +prepare_losoto_bandpass.control.smooth.replace = False +prepare_losoto_bandpass.control.bandpass.operation = SMOOTH +prepare_losoto_bandpass.control.bandpass.soltab = sol000/amplitude000 +prepare_losoto_bandpass.control.bandpass.pol = XX,YY +prepare_losoto_bandpass.control.bandpass.axesToSmooth = [freq] +prepare_losoto_bandpass.control.bandpass.mode = savitzky-golay +prepare_losoto_bandpass.control.bandpass.size = [17] +prepare_losoto_bandpass.control.bandpass.degree = 2 +prepare_losoto_bandpass.control.bandpass.log = True +prepare_losoto_bandpass.control.interp.operation = INTERPOLATE +prepare_losoto_bandpass.control.interp.soltab = sol000/amplitude000 +prepare_losoto_bandpass.control.interp.outSoltab = bandpass +prepare_losoto_bandpass.control.interp.axisToRegrid = freq +prepare_losoto_bandpass.control.interp.maxFlaggedWidth = {{ max2interpolate }} +prepare_losoto_bandpass.control.interp.newDelta = {{ bandpass_freqresolution }} +prepare_losoto_bandpass.control.interp.delta = {{ avg_freqresolution }} +prepare_losoto_bandpass.control.interp.log = True +prepare_losoto_bandpass.control.smoothb.operation = SMOOTH +prepare_losoto_bandpass.control.smoothb.soltab = sol000/bandpass +prepare_losoto_bandpass.control.smoothb.pol = XX,YY +prepare_losoto_bandpass.control.smoothb.axesToSmooth = [time] +prepare_losoto_bandpass.control.smoothb.mode = median +prepare_losoto_bandpass.control.smoothb.log = True +prepare_losoto_bandpass.control.smoothb.replace = True +prepare_losoto_bandpass.control.plotB1.operation = PLOT +prepare_losoto_bandpass.control.plotB1.soltab = sol000/bandpass +prepare_losoto_bandpass.control.plotB1.axesInPlot = [time,freq] +prepare_losoto_bandpass.control.plotB1.axisInTable = ant +prepare_losoto_bandpass.control.plotB1.plotFlag = True +prepare_losoto_bandpass.control.plotB1.prefix = {{ inspection_directory }}/bandpass_ +prepare_losoto_bandpass.control.plotB2.operation = PLOT +prepare_losoto_bandpass.control.plotB2.soltab = sol000/bandpass +prepare_losoto_bandpass.control.plotB2.axesInPlot = freq +prepare_losoto_bandpass.control.plotB2.axisInTable = ant +prepare_losoto_bandpass.control.plotB2.axisInCol = pol +prepare_losoto_bandpass.control.plotB2.plotFlag = True +prepare_losoto_bandpass.control.plotB2.prefix = {{ inspection_directory }}/bandpass_ +prepare_losoto_bandpass.control.plotB2.time.minmaxstep = [0,1e20,500000] +prepare_losoto_bandpass.control.plotB3.operation = PLOT +prepare_losoto_bandpass.control.plotB3.soltab = sol000/bandpass +prepare_losoto_bandpass.control.plotB3.axesInPlot = freq +prepare_losoto_bandpass.control.plotB3.axisInCol = ant +prepare_losoto_bandpass.control.plotB3.plotFlag = True +prepare_losoto_bandpass.control.plotB3.prefix = {{ inspection_directory }}/bandpass_ +prepare_losoto_bandpass.control.plotB3.time.minmaxstep = [0,1e20,500000] + +# create losoto v2 parset file +prepare_losoto_bandpasstrans.control.kind = plugin +prepare_losoto_bandpasstrans.control.type = makeLosotoParset +prepare_losoto_bandpasstrans.control.steps = [plotB1,plotB2,plotB3] +prepare_losoto_bandpasstrans.control.filename = {{ job_directory }}/losoto_transfer.parset +prepare_losoto_bandpasstrans.control.global.ncpu = {{ num_proc_per_node }} +prepare_losoto_bandpasstrans.control.plotB1.operation = PLOT +prepare_losoto_bandpasstrans.control.plotB1.soltab = calibrator/bandpass +prepare_losoto_bandpasstrans.control.plotB1.axesInPlot = [time,freq] +prepare_losoto_bandpasstrans.control.plotB1.axisInTable = ant +prepare_losoto_bandpasstrans.control.plotB1.plotFlag = True +prepare_losoto_bandpasstrans.control.plotB1.prefix = {{ inspection_directory }}/bandpass_transfer_ +prepare_losoto_bandpasstrans.control.plotB2.operation = PLOT +prepare_losoto_bandpasstrans.control.plotB2.soltab = calibrator/bandpass +prepare_losoto_bandpasstrans.control.plotB2.axesInPlot = freq +prepare_losoto_bandpasstrans.control.plotB2.axisInTable = ant +prepare_losoto_bandpasstrans.control.plotB2.axisInCol = pol +prepare_losoto_bandpasstrans.control.plotB2.plotFlag = True +prepare_losoto_bandpasstrans.control.plotB2.prefix = {{ inspection_directory }}/bandpass_transfer_ +prepare_losoto_bandpasstrans.control.plotB2.time.minmaxstep = [0,1e20,500000] +prepare_losoto_bandpasstrans.control.plotB3.operation = PLOT +prepare_losoto_bandpasstrans.control.plotB3.soltab = calibrator/bandpass +prepare_losoto_bandpasstrans.control.plotB3.axesInPlot = freq +prepare_losoto_bandpasstrans.control.plotB3.axisInCol = ant +prepare_losoto_bandpasstrans.control.plotB3.plotFlag = True +prepare_losoto_bandpasstrans.control.plotB3.prefix = {{ inspection_directory }}/bandpass_transfer_ +prepare_losoto_bandpasstrans.control.plotB3.time.minmaxstep = [0,1e20,500000] + +# do the processing on the LoSoTo file +process_losoto_bandpass.control.kind = recipe +process_losoto_bandpass.control.type = executable_args +process_losoto_bandpass.control.inplace = True +process_losoto_bandpass.control.executable = {{ losoto_directory }}/bin/losoto +process_losoto_bandpass.control.max_per_node = {{ num_proc_per_node }} +process_losoto_bandpass.control.mapfile_in = h5imp_cal_bandpass.output.mapfile +process_losoto_bandpass.control.inputkey = h5in +process_losoto_bandpass.argument.flags = [-v,h5in,{{ job_directory }}/losoto.parset] + +# output the final soltab into an external h5parm +h5exp_cal_bandpass.control.kind = recipe +h5exp_cal_bandpass.control.type = executable_args +h5exp_cal_bandpass.control.inplace = True +h5exp_cal_bandpass.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py +h5exp_cal_bandpass.control.error_tolerance = {{ error_tolerance }} +h5exp_cal_bandpass.control.mapfile_in = h5imp_cal_bandpass.output.mapfile +h5exp_cal_bandpass.control.inputkey = h5in +h5exp_cal_bandpass.argument.flags = [-q,-v,h5in] +h5exp_cal_bandpass.argument.insoltab = bandpass +h5exp_cal_bandpass.argument.outh5parm = {{ cal_solutions }} +h5exp_cal_bandpass.argument.outsolset = calibrator + +# transfer solutions if necessary +transfer_solutions.control.type = pythonplugin +transfer_solutions.control.executable = {{ scripts }}/transfer_solutions.py +transfer_solutions.control.error_tolerance = {{ error_tolerance }} +transfer_solutions.control.skip_infile = True +transfer_solutions.control.mapfile_in = sky_cal.output.SkymodelName.mapfile +transfer_solutions.argument.flags = [{{ cal_solutions }}, {{ solutions2transfer }}] +transfer_solutions.argument.insolset = calibrator +transfer_solutions.argument.outsolset = calibrator +transfer_solutions.argument.insoltab = bandpass +transfer_solutions.argument.outsoltab = bandpass +transfer_solutions.argument.antenna = {{ antennas2transfer }} +transfer_solutions.argument.parset = {{ job_directory }}/losoto_transfer.parset + +################################ +## calibrate ionosphere ## +################################ +# collect all instrument tables into one h5parm +h5imp_cal_ion.control.kind = recipe +h5imp_cal_ion.control.type = executable_args +h5imp_cal_ion.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py +h5imp_cal_ion.control.error_tolerance = {{ error_tolerance }} +h5imp_cal_ion.control.mapfile_in = h5imp_cal_map.output.mapfile +h5imp_cal_ion.control.inputkey = h5parm +h5imp_cal_ion.control.outputkey = outh5parm +h5imp_cal_ion.argument.flags = [-q,-v,-c,h5parm] +h5imp_cal_ion.argument.outh5parm = outh5parm + +# create losoto v2 parset file +prepare_losoto_ion.control.kind = plugin +prepare_losoto_ion.control.type = makeLosotoParset +prepare_losoto_ion.control.steps = [plotA1,flag,flagextend,plotA2,merge,plotP3,plotPd,duplicatePbkp,{{ cal_ion }},plotClock,plotPr,plotPr3,flagstation] +prepare_losoto_ion.control.filename = {{ job_directory }}/losoto.parset +prepare_losoto_ion.control.global.ncpu = {{ num_proc_per_node }} +prepare_losoto_ion.control.plotA1.operation = PLOT +prepare_losoto_ion.control.plotA1.soltab = sol000/amplitude000 +prepare_losoto_ion.control.plotA1.axesInPlot = [time,freq] +prepare_losoto_ion.control.plotA1.axisInTable = ant +prepare_losoto_ion.control.plotA1.plotFlag = True +prepare_losoto_ion.control.plotA1.prefix = {{ inspection_directory }}/ion_ampBFlag_ +prepare_losoto_ion.control.flag.operation = FLAG +prepare_losoto_ion.control.flag.soltab = sol000/amplitude000 +prepare_losoto_ion.control.flag.axesToFlag = [time,freq] +prepare_losoto_ion.control.flag.order = [100,40] +prepare_losoto_ion.control.flag.maxCycles = 1 +prepare_losoto_ion.control.flag.maxRms = 5 +prepare_losoto_ion.control.flag.replace = False +prepare_losoto_ion.control.flag.preFlagZeros = False +prepare_losoto_ion.control.flag.mode = smooth +prepare_losoto_ion.control.flagextend.operation = FLAGEXTEND +prepare_losoto_ion.control.flagextend.soltab = sol000/amplitude000 +prepare_losoto_ion.control.flagextend.axesToExt = [time,freq] +prepare_losoto_ion.control.flagextend.size = [200,80] +prepare_losoto_ion.control.flagextend.percent = 50 +prepare_losoto_ion.control.flagextend.maxCycles = 2 +prepare_losoto_ion.control.plotA2.operation = PLOT +prepare_losoto_ion.control.plotA2.soltab = sol000/amplitude000 +prepare_losoto_ion.control.plotA2.axesInPlot = [time,freq] +prepare_losoto_ion.control.plotA2.axisInTable = ant +prepare_losoto_ion.control.plotA2.plotFlag = True +prepare_losoto_ion.control.plotA2.prefix = {{ inspection_directory }}/ion_ampAFlag_ +prepare_losoto_ion.control.merge.operation = REWEIGHT +prepare_losoto_ion.control.merge.mode = copy +prepare_losoto_ion.control.merge.soltab = sol000/phase000 +prepare_losoto_ion.control.merge.soltabImport = amplitude000 +prepare_losoto_ion.control.plotP3.operation = PLOT +prepare_losoto_ion.control.plotP3.soltab = sol000/phase000 +prepare_losoto_ion.control.plotP3.axesInPlot = [time,freq] +prepare_losoto_ion.control.plotP3.axisInTable = ant +prepare_losoto_ion.control.plotP3.plotFlag = True +prepare_losoto_ion.control.plotP3.prefix = {{ inspection_directory }}/ion_ph_ +prepare_losoto_ion.control.plotP3.refAnt = {{ refant }} +prepare_losoto_ion.control.plotP3.minmax = [-3.14,3.14] +prepare_losoto_ion.control.plotPd.operation = PLOT +prepare_losoto_ion.control.plotPd.soltab = sol000/phase000 +prepare_losoto_ion.control.plotPd.axesInPlot = [time,freq] +prepare_losoto_ion.control.plotPd.axisInTable = ant +prepare_losoto_ion.control.plotPd.axisDiff = pol +prepare_losoto_ion.control.plotPd.plotFlag = True +prepare_losoto_ion.control.plotPd.prefix = {{ inspection_directory }}/ion_ph_poldif +prepare_losoto_ion.control.plotPd.refAnt = {{ refant }} +prepare_losoto_ion.control.plotPd.minmax = [-3.14,3.14] +prepare_losoto_ion.control.duplicatePbkp.operation = DUPLICATE +prepare_losoto_ion.control.duplicatePbkp.soltab = sol000/phase000 +prepare_losoto_ion.control.duplicatePbkp.soltabOut = phaseOrig +prepare_losoto_ion.control.ct.operation = CLOCKTEC +prepare_losoto_ion.control.ct.soltab = sol000/phase000 +prepare_losoto_ion.control.ct.clocksoltabOut = clock +prepare_losoto_ion.control.ct.tecsoltabOut = tec +prepare_losoto_ion.control.ct.tec3rdsoltabOut = tec3rd +prepare_losoto_ion.control.ct.offsetsoltabOut = phase_offset +prepare_losoto_ion.control.ct.CombinePol = True +prepare_losoto_ion.control.ct.FlagBadChannels = False +prepare_losoto_ion.control.ct.Fit3rdOrder = False +prepare_losoto_ion.control.ct.Circular = False +prepare_losoto_ion.control.ct3.operation = CLOCKTEC +prepare_losoto_ion.control.ct3.soltab = sol000/phase000 +prepare_losoto_ion.control.ct3.clocksoltabOut = clock +prepare_losoto_ion.control.ct3.tecsoltabOut = tec +prepare_losoto_ion.control.ct3.tec3rdsoltabOut = tec3rd +prepare_losoto_ion.control.ct3.offsetsoltabOut = phase_offset +prepare_losoto_ion.control.ct3.CombinePol = True +prepare_losoto_ion.control.ct3.FlagBadChannels = False +prepare_losoto_ion.control.ct3.Fit3rdOrder = True +prepare_losoto_ion.control.ct3.Circular = False +prepare_losoto_ion.control.plotClock.operation = PLOT +prepare_losoto_ion.control.plotClock.soltab = sol000/clock +prepare_losoto_ion.control.plotClock.axesInPlot = [time] +prepare_losoto_ion.control.plotClock.axisInTable = ant +prepare_losoto_ion.control.plotClock.prefix = {{ inspection_directory }}/clock +prepare_losoto_ion.control.plotClock.plotFlag = False +prepare_losoto_ion.control.plotClock.refAnt = {{ refant }} +prepare_losoto_ion.control.plotTEC.operation = PLOT +prepare_losoto_ion.control.plotTEC.soltab = sol000/tec +prepare_losoto_ion.control.plotTEC.axesInPlot = [time] +prepare_losoto_ion.control.plotTEC.axisInTable = ant +prepare_losoto_ion.control.plotTEC.prefix = {{ inspection_directory }}/tec +prepare_losoto_ion.control.plotTEC.plotFlag = False +prepare_losoto_ion.control.plotTEC.refAnt = {{ refant }} +prepare_losoto_ion.control.plotTEC3.operation = PLOT +prepare_losoto_ion.control.plotTEC3.soltab = sol000/tec3rd +prepare_losoto_ion.control.plotTEC3.axesInPlot = [time] +prepare_losoto_ion.control.plotTEC3.axisInTable = ant +prepare_losoto_ion.control.plotTEC3.prefix = {{ inspection_directory }}/tec3rd +prepare_losoto_ion.control.plotTEC3.plotFlag = False +prepare_losoto_ion.control.plotTEC3.refAnt = {{ refant }} +prepare_losoto_ion.control.smooth.operation = SMOOTH +prepare_losoto_ion.control.smooth.soltab = sol000/clock +prepare_losoto_ion.control.smooth.axesToSmooth = [time] +prepare_losoto_ion.control.smooth.mode = median +prepare_losoto_ion.control.smooth.log = False +prepare_losoto_ion.control.smooth.replace = True +prepare_losoto_ion.control.residuals.operation = RESIDUALS +prepare_losoto_ion.control.residuals.soltab = sol000/phase000 +prepare_losoto_ion.control.residuals.soltabsToSub = [tec,clock] +prepare_losoto_ion.control.residuals3.operation = RESIDUALS +prepare_losoto_ion.control.residuals3.soltab = sol000/phase000 +prepare_losoto_ion.control.residuals3.soltabsToSub = [tec,clock,tec3rd] # only for very low-freq dataset +prepare_losoto_ion.control.plotPr.operation = PLOT +prepare_losoto_ion.control.plotPr.soltab = sol000/phase000 +prepare_losoto_ion.control.plotPr.axesInPlot = [time,freq] +prepare_losoto_ion.control.plotPr.axisInTable = ant +prepare_losoto_ion.control.plotPr.axisDiff = pol +prepare_losoto_ion.control.plotPr.plotFlag = True +prepare_losoto_ion.control.plotPr.prefix = {{ inspection_directory }}/ion_ph-res_poldif +prepare_losoto_ion.control.plotPr.refAnt = {{ refant }} +prepare_losoto_ion.control.plotPr.minmax = [-3.14,3.14] +prepare_losoto_ion.control.plotPr3.operation = PLOT +prepare_losoto_ion.control.plotPr3.soltab = sol000/phase000 +prepare_losoto_ion.control.plotPr3.axesInPlot = [time,freq] +prepare_losoto_ion.control.plotPr3.axisInTable = ant +prepare_losoto_ion.control.plotPr3.plotFlag = True +prepare_losoto_ion.control.plotPr3.prefix = {{ inspection_directory }}/ion_ph-res_ +prepare_losoto_ion.control.plotPr3.refAnt = {{ refant }} +prepare_losoto_ion.control.plotPr3.minmax = [-3.14,3.14] +prepare_losoto_ion.control.flagstation.operation = FLAGSTATION +prepare_losoto_ion.control.flagstation.soltab = sol000/phase000 +prepare_losoto_ion.control.flagstation.maxStddev = {{ maxStddev }} +prepare_losoto_ion.control.flagstation.mode = resid +prepare_losoto_ion.control.flagstation.refAnt = {{ refant }} +prepare_losoto_ion.control.flagstation.soltabExport = clock + +# do the processing on the LoSoTo file +process_losoto_ion.control.kind = recipe +process_losoto_ion.control.type = executable_args +process_losoto_ion.control.inplace = True +process_losoto_ion.control.executable = {{ losoto_directory }}/bin/losoto +process_losoto_ion.control.max_per_node = {{ num_proc_per_node }} +process_losoto_ion.control.mapfile_in = h5imp_cal_ion.output.mapfile +process_losoto_ion.control.inputkey = h5in +process_losoto_ion.argument.flags = [-v,h5in,{{ job_directory }}/losoto.parset] + +# output the final soltab into an external h5parm +h5exp_cal_ion.control.kind = recipe +h5exp_cal_ion.control.type = executable_args +h5exp_cal_ion.control.inplace = True +h5exp_cal_ion.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py +h5exp_cal_ion.control.error_tolerance = {{ error_tolerance }} +h5exp_cal_ion.control.mapfile_in = h5imp_cal_ion.output.mapfile +h5exp_cal_ion.control.inputkey = h5in +h5exp_cal_ion.argument.flags = [-q,-v,h5in] +h5exp_cal_ion.argument.insoltab = {{ tables2export }} +h5exp_cal_ion.argument.outh5parm = {{ cal_solutions }} +h5exp_cal_ion.argument.outsolset = calibrator + + +################################ +## applying the results ## +################################ +# apply the PA solutions +apply_PA.control.type = dppp +apply_PA.control.error_tolerance = {{ error_tolerance }} +apply_PA.control.inplace = True +apply_PA.control.max_per_node = {{ num_proc_per_node_limit }} +apply_PA.control.mapfile_in = ndppp_prep_cal.output.mapfile +apply_PA.control.inputkey = msfile +apply_PA.argument.msin = msfile +apply_PA.argument.numthreads = {{ max_dppp_threads }} +apply_PA.argument.msin.datacolumn = DATA +apply_PA.argument.msout.datacolumn = CORRECTED_DATA +apply_PA.argument.msout.storagemanager = "Dysco" +apply_PA.argument.msout.storagemanager.databitrate = 0 +apply_PA.argument.steps = [applyPA] +apply_PA.argument.applyPA.type = applycal +apply_PA.argument.applyPA.correction = polalign +apply_PA.argument.applyPA.parmdb = {{ cal_solutions }} + +# apply the bandpass +apply_bandpass.control.type = dppp +apply_bandpass.control.error_tolerance = {{ error_tolerance }} +apply_bandpass.control.inplace = True +apply_bandpass.control.max_per_node = {{ num_proc_per_node_limit }} +apply_bandpass.control.mapfile_in = ndppp_prep_cal.output.mapfile +apply_bandpass.control.inputkey = msfile +apply_bandpass.argument.msin = msfile +apply_bandpass.argument.numthreads = {{ max_dppp_threads }} +apply_bandpass.argument.msin.datacolumn = CORRECTED_DATA +apply_bandpass.argument.msout.datacolumn = CORRECTED_DATA +apply_bandpass.argument.msout.storagemanager = "Dysco" +apply_bandpass.argument.msout.storagemanager.databitrate = 0 +apply_bandpass.argument.steps = [applybandpass] +apply_bandpass.argument.applybandpass.type = applycal +apply_bandpass.argument.applybandpass.correction = bandpass +apply_bandpass.argument.applybandpass.parmdb = {{ cal_solutions }} +apply_bandpass.argument.applybandpass.updateweights = True + +# apply the beam +apply_beam.control.type = dppp +apply_beam.control.error_tolerance = {{ error_tolerance }} +apply_beam.control.inplace = True +apply_beam.control.max_per_node = {{ num_proc_per_node_limit }} +apply_beam.control.mapfile_in = ndppp_prep_cal.output.mapfile +apply_beam.control.inputkey = msfile +apply_beam.argument.msin = msfile +apply_beam.argument.numthreads = {{ max_dppp_threads }} +apply_beam.argument.msin.datacolumn = CORRECTED_DATA +apply_beam.argument.msout.datacolumn = CORRECTED_DATA +apply_beam.argument.msout.storagemanager = "Dysco" +apply_beam.argument.msout.storagemanager.databitrate = 0 +apply_beam.argument.steps = [applybeam] +apply_beam.argument.applybeam.type = applybeam +apply_beam.argument.applybeam.invert = True +apply_beam.argument.applybeam.usechannelfreq = False +apply_beam.argument.applybeam.beammode = element +apply_beam.argument.applybeam.updateweights = True + +# apply the FR solutions +apply_FR.control.type = dppp +apply_FR.control.error_tolerance = {{ error_tolerance }} +apply_FR.control.inplace = True +apply_FR.control.max_per_node = {{ num_proc_per_node_limit }} +apply_FR.control.mapfile_in = ndppp_prep_cal.output.mapfile +apply_FR.control.inputkey = msfile +apply_FR.argument.msin = msfile +apply_FR.argument.numthreads = {{ max_dppp_threads }} +apply_FR.argument.msin.datacolumn = CORRECTED_DATA +apply_FR.argument.msout.datacolumn = CORRECTED_DATA +apply_FR.argument.msout.storagemanager = "Dysco" +apply_FR.argument.msout.storagemanager.databitrate = 0 +apply_FR.argument.steps = [applyFR] +apply_FR.argument.applyFR.type = applycal +apply_FR.argument.applyFR.correction = faraday +apply_FR.argument.applyFR.parmdb = {{ cal_solutions }} + +# apply the clock solutions +apply_clock.control.type = dppp +apply_clock.control.error_tolerance = {{ error_tolerance }} +apply_clock.control.inplace = True +apply_clock.control.max_per_node = {{ num_proc_per_node_limit }} +apply_clock.control.mapfile_in = ndppp_prep_cal.output.mapfile +apply_clock.control.inputkey = msfile +apply_clock.argument.msin = msfile +apply_clock.argument.numthreads = {{ max_dppp_threads }} +apply_clock.argument.msin.datacolumn = CORRECTED_DATA +apply_clock.argument.msout.datacolumn = CORRECTED_DATA +apply_clock.argument.msout.storagemanager = "Dysco" +apply_clock.argument.msout.storagemanager.databitrate = 0 +apply_clock.argument.steps = [applyclock] +apply_clock.argument.applyclock.type = applycal +apply_clock.argument.applyclock.correction = clock +apply_clock.argument.applyclock.parmdb = {{ cal_solutions }} + +# apply the TEC solutions +apply_TEC.control.type = dppp +apply_TEC.control.error_tolerance = {{ error_tolerance }} +apply_TEC.control.inplace = True +apply_TEC.control.max_per_node = {{ num_proc_per_node_limit }} +apply_TEC.control.mapfile_in = ndppp_prep_cal.output.mapfile +apply_TEC.control.inputkey = msfile +apply_TEC.argument.msin = msfile +apply_TEC.argument.numthreads = {{ max_dppp_threads }} +apply_TEC.argument.msin.datacolumn = CORRECTED_DATA +apply_TEC.argument.msout.datacolumn = CORRECTED_DATA +apply_TEC.argument.msout.storagemanager = "Dysco" +apply_TEC.argument.msout.storagemanager.databitrate = 0 +apply_TEC.argument.steps = [applytec] +apply_TEC.argument.applytec.type = applycal +apply_TEC.argument.applytec.correction = tec +apply_TEC.argument.applytec.parmdb = {{ cal_solutions }} + +# # apply the phase offset solutions +# apply_offset.control.type = dppp +# apply_offset.control.error_tolerance = {{ error_tolerance }} +# apply_offset.control.inplace = True +# apply_offset.control.max_per_node = {{ num_proc_per_node_limit }} +# apply_offset.control.mapfile_in = ndppp_prep_cal.output.mapfile +# apply_offset.control.inputkey = msfile +# apply_offset.argument.msin = msfile +# apply_offset.argument.numthreads = {{ max_dppp_threads }} +# apply_offset.argument.msin.datacolumn = CORRECTED_DATA +# apply_offset.argument.msout.datacolumn = CORRECTED_DATA +# apply_offset.argument.msout.storagemanager = "Dysco" +# apply_offset.argument.msout.storagemanager.databitrate = 0 +# apply_offset.argument.steps = [applyoffset] +# apply_offset.argument.applyoffset.type = applycal +# apply_offset.argument.applyoffset.correction = phase_offset +# apply_offset.argument.applyoffset.parmdb = {{ cal_solutions }} + +############################### +## finalizing the results ## +################################ +# set the pointing direction +h5parm_name.control.type = pythonplugin +h5parm_name.control.executable = {{ scripts }}/h5parm_pointingname.py +h5parm_name.control.error_tolerance = {{ error_tolerance }} +h5parm_name.argument.flags = [{{ cal_solutions }}] +h5parm_name.argument.solsetName = calibrator +h5parm_name.argument.pointing = sky_cal.output.SkymodelName.mapfile + +# set the pointing direction +make_summary.control.type = pythonplugin +make_summary.control.executable = {{ scripts }}/make_summary.py +make_summary.control.error_tolerance = {{ error_tolerance }} +make_summary.control.mapfile_in = combine_data_cal_map.output.mapfile +make_summary.control.inputkey = infiles +make_summary.argument.observation_directory = {{ working_directory }} +make_summary.argument.logfile = {{ log_file }} +make_summary.argument.h5parmdb = {{ cal_solutions }} +make_summary.argument.inspection_directory = {{ inspection_directory }} +make_summary.argument.MSfile = infiles + +######################################################## +## ## +## END PIPELINE ## +## ## +######################################################## diff --git a/prefactor3/Pre-Facet-Target.parset b/prefactor3/Pre-Facet-Target.parset new file mode 100644 index 0000000000000000000000000000000000000000..94d0fc7bab69b11def2f605f8e21c8084f49c982 --- /dev/null +++ b/prefactor3/Pre-Facet-Target.parset @@ -0,0 +1,893 @@ +########################################################################## +# Pre-Facet Target Calibration Pipeline v3.0 (04/09/2019) # +# # +# Target part of the basic Pre-Facet calibration pipeline: # +# - requires LOFAR software version >= 3.1.0 # +# - requires losoto software version >= 2.0.0 # +# - expects shared filesystem, that all nodes can reach all files! # +# (E.g. a single workstation or compute cluster with shared filesystem # +# doesn't work on multiple nodes on CEP3.) # +########################################################################## + +########################################## +### parameters you will need to adjust. ## +########################################## + +## information about the target data +! target_input_path = /input_data/target ## specify the directory where your target data is stored +! target_input_pattern = *.MS ## regular expression pattern of all your target files + +## location of the software +! prefactor_directory = /opt/prefactor/ ## path to your prefactor copy +! losoto_directory = /opt/lofarsoft ## path to your local LoSoTo installation +! aoflagger = /opt/lofarsoft/bin/aoflagger ## path to your aoflagger executable + +## location of the calibrator solutions +! cal_solutions = /output_data/Pre-Facet-Calibrator/results/cal_values/cal_solutions.h5 + +########################################## +### parameters you may need to adjust ## +########################################## + +! refant = 'CS001HBA0' ## name of the station that will be used as a reference for the phase plots, 'closest' will reference to the spatially closest unflagged antenna +! flag_baselines = [] ## NDPPP-compatible pattern for baselines or stations to be flagged (may be an empty list, i.e.: [] ) +! process_baselines_target = [CR]S*& ## performs A-Team-clipping/demixing and direction-independent phase-only self-calibration only on these baselines. Choose [CR]S*& if you want to process only cross-correlations and remove international stations. +! filter_baselines = {{ process_baselines_target }} ## selects only this set of baselines to be processed for the full pipeline. Choose [CR]S*& if you want to process only cross-correlations and remove international stations. +! do_smooth = False ## enable or disable baseline-based smoothing +! rfistrategy = HBAdefault.rfis ## strategy to be applied with the statistical flagger (AOFlagger) for wideband flagging +! min_unflagged_fraction = 0.5 ## minimum fraction of unflagged data after RFI flagging and A-team clipping +! compression_bitrate = 16 ## defines the bitrate of Dysco compression of the data after the final step, choose 0 if you do NOT want to compress the data +! raw_data = False ## use autoweight, set to True in case you are using raw data +! propagatesolutions = True ## use already derived solutions as initial guess for the upcoming time slot + +# demixing options (only used if demix step is added to the prep_cal_strategy variable) +! demix_sources = [CasA,CygA] ## choose sources to demix (provided as list) +! demix_target = "" ## if given, the target source model (its patch in the SourceDB) is taken into account when solving +! demix_freqstep = 16 ## number of channels to average when demixing. +! demix_timestep = 10 ## number of time slots to average when demixing + +# definitions for pipeline options -- do not change! +! default_flagging = flagbaseline,flagelev,flagamp ## regular flagging after pre-processing by the observatory pipelines +! raw_flagging = flagedge,aoflag,{{ default_flagging }} ## full flagging (usually only necessary for raw data) +! demix = demix, ## Do not change! Only demix_step should be edited if needed +! clipATeam = clipATeam, ## Do not change! Only clipATeam_step should be edited if needed +! none = ## Do not change! + +# pipeline options +! initial_flagging = {{ default_flagging }} ## choose {{ raw_flagging }} if you process raw data +! demix_step = {{ none }} ## choose {{ demix }} if you want to demix +! apply_steps = applyclock,applybeam,applyRM ## comma-separated list of apply_steps performed in the target preparation (NOTE: only use applyRM if you have performed RMextract before!) +! clipATeam_step = {{ clipATeam }} ## choose {{ none }} if you want to skip A-team-clipping +! gsmcal_step = phase ## choose tec if you want to fit TEC instead of self-calibrating for phases +! updateweights = True ## update the weights column, in a way consistent with the weights being inverse proportional to the autocorrelations + +########################################## +### parameters for pipeline performance ## +########################################## + +! num_proc_per_node = input.output.max_per_node ## number of processes to use per step per node (usually max_per_node from pipeline.cfg) +! num_proc_per_node_limit = 4 ## number of processes to use per step per node for tasks with high i/o (dppp or cp) or memory (eg calibration) +! max_dppp_threads = 10 ## number of threads per process for NDPPP +! min_length = 5 ## minimum amount of chunks to concatenate in frequency necessary to perform the wide-band flagging in the RAM. It data is too big aoflag will use indirect-read. +! overhead = 0.7 ## Only use this fraction of the available memory for deriving the amount of data to be concatenated. +! min_separation = 30 ## minimal accepted distance to an A-team source on the sky in degrees (will raise a WARNING) + +! error_tolerance = False ## set this to True if you want the pipeline run to continue if single bands fail + +########################################## +### parameters you may want to adjust ## +########################################## + +## main directories +! lofar_directory = $LOFARROOT ## base directory of your LOFAR installation +! job_directory = input.output.job_directory ## directory of the prefactor outputs +! working_directory = input.output.working_directory/input.output.job_name ## specify the working_directory (intermediate data products) +! log_file = input.output.log_file ## location of the logfile +! mapfile_dir = input.output.mapfile_dir ## specify mapfile directory + +## script and plugin directories +! scripts = {{ prefactor_directory }}/scripts +pipeline.pluginpath = {{ prefactor_directory }}/plugins + +## skymodel directory +! calibrator_path_skymodel = {{ prefactor_directory }}/skymodels +! A-team_skymodel = {{ calibrator_path_skymodel }}/Ateam_LBA_CC.skymodel +! target_skymodel = {{ job_directory }}/target.skymodel ## path to the skymodel for the phase-only calibration of the target +! use_target = True ## download the phase-only calibration skymodel from TGSS, "Force" : always download , "True" download if {{ target_skymodel }} does not exist , "False" : never download +! skymodel_source = TGSS ## use GSM if you want to use the experimental (!) GSM SkyModel creator using TGSS, NVSS, WENSS and VLSS + +## result directories +! results_directory = {{ job_directory }}/results ## location of the results +! inspection_directory = {{ results_directory }}/inspection ## directory where the inspection plots will be stored +! cal_values_directory = {{ results_directory }}/cal_values ## directory where the final h5parm solution set will be stored + +## calibrator + target solutions +! solutions = {{ cal_values_directory }}/solutions.h5 + +## averaging for the target data +! avg_timeresolution = 4. ## average to 4 sec/timeslot +! avg_freqresolution = 48.82kHz ## average to 48.82 kHz/ch (= 4 ch/SB) +! avg_timeresolution_concat = 8. ## average to 8 sec/timeslot +! avg_freqresolution_concat = 97.64kHz ## average to 97.64 kHz/ch (= 2 ch/SB) + +## concatenating the target data +! num_SBs_per_group = 10 ## make concatenated measurement-sets with that many subbands, choose a high number if running LBA +! reference_stationSB = None ## station-subband number to use as reference for grouping, "None" -> use lowest frequency input data as reference + +## RMextract settings +! ionex_server = "ftp://ftp.aiub.unibe.ch/CODE/" ## to download from the "standard" server +! ionex_prefix = CODG ## the prefix of the IONEX files +! ionex_path = {{ job_directory }}/IONEX/ ## path where the IONEX files can be stored or are already stored +## Proxy Settings for RMextract ## Only needed if commmunication to the outside world goes via proxy, leave empty otherwise +! proxy_server = ## Url "my.proxy.com" or ip of proxy server +! proxy_port = ## Port of the server +! proxy_type = ## Proxy Type: "socks4" or "socks5" +! proxy_user = None ## username for proxy server. Leave None if you do not need one +! proxy_pass = None ## Password for proxy server. Leave None if you do not need one + +######################################################## +## ## +## BEGIN PIPELINE: DO NOT UPDATE BELOW THIS LINE! ## +## ## +######################################################## + +# which steps to run +pipeline.steps = [prep, {{ clipATeam_step }} concat, prep_gsmcal, {{ gsmcal_step }}, finalize] + +# pipeline substeps +pipeline.steps.prep = [createmap_target, get_targetname, combine_data_target_map, check_Ateam_separation, mk_targ_values_dir, copy_cal_sols, check_station_mismatch, createmap_preptarg, createmap_insttarg, create_ateam_model_map, make_sourcedb_ateam, expand_sourcedb_ateam, h5imp_RMextract, prepare_losoto_RMextract, process_losoto_RMextract, ndppp_prep_target] + +pipeline.steps.clipATeam = [predict_ateam, ateamcliptar, plotateamclip] +pipeline.steps.concat = [combine_target_map, check_bad_antennas, sortmap_target, do_sortmap_maps, dpppconcat, combine_concat_map, ms_concat_target, ms_concat_target_map, expand_memory_map, aoflag] +pipeline.steps.prep_gsmcal = [check_unflagged, check_unflagged_map, combine_concat_map, combine_frac_map, plot_unflagged, sky_tar, create_target_model_map, make_sourcedb_target, expand_sourcedb_target, gsmcal_parmmap, h5_gsmsol_map, smooth_data] +pipeline.steps.phase = [gsmcal_phase, h5imp_gsmcal, prepare_losoto_phase] +pipeline.steps.tec = [gsmcal_tec, h5imp_gsmcal, prepare_losoto_tec ] + +pipeline.steps.finalize = [process_losoto_gsmcal, add_missing_stations, h5exp_gsm, apply_gsmcal, make_results_mapfile, make_results_compress, move_results, h5parm_name, structure_function, make_summary] + + +############################# +## Prepare target part ## +############################# +# generate a mapfile of all the target data +createmap_target.control.kind = plugin +createmap_target.control.type = createMapfile +createmap_target.control.method = mapfile_from_folder +createmap_target.control.mapfile_dir = {{ mapfile_dir }} +createmap_target.control.filename = createmap_target.mapfile +createmap_target.control.folder = {{ target_input_path }} +createmap_target.control.pattern = {{ target_input_pattern }} + +# get the target name +get_targetname.control.kind = plugin +get_targetname.control.type = getTargetName +get_targetname.control.mapfile_in = createmap_target.output.mapfile + +# combine all entries into one mapfile, for the sortmap script +combine_data_target_map.control.kind = plugin +combine_data_target_map.control.type = createMapfile +combine_data_target_map.control.method = mapfile_all_to_one +combine_data_target_map.control.mapfile_dir = {{ mapfile_dir }} +combine_data_target_map.control.filename = combine_data_tar_map.mapfile +combine_data_target_map.control.mapfile_in = createmap_target.output.mapfile + +# warn for potential nearby A-Team sources +check_Ateam_separation.control.type = pythonplugin +check_Ateam_separation.control.executable = {{ scripts }}/check_Ateam_separation.py +check_Ateam_separation.control.mapfile_in = combine_data_target_map.output.mapfile +check_Ateam_separation.control.inputkey = MSfile +check_Ateam_separation.argument.min_separation = {{ min_separation }} +check_Ateam_separation.argument.outputimage = {{ inspection_directory }}/A-Team_elevation_target.png +check_Ateam_separation.argument.flags = [MSfile] + +# create the cal_values_directory if needed +mk_targ_values_dir.control.kind = plugin +mk_targ_values_dir.control.type = makeDirectory +mk_targ_values_dir.control.directory = {{ cal_values_directory }} + +# move the results to where we want them +copy_cal_sols.control.kind = recipe +copy_cal_sols.control.type = executable_args +copy_cal_sols.control.executable = /bin/cp +copy_cal_sols.control.max_per_node = 1 +copy_cal_sols.control.skip_infile = True +copy_cal_sols.control.mapfile_in = combine_data_target_map.output.mapfile +copy_cal_sols.argument.flags = [{{ cal_solutions }},{{ solutions }}] + +# check potential station mismatch +check_station_mismatch.control.kind = plugin +check_station_mismatch.control.type = compareStationList +check_station_mismatch.control.mapfile_in = createmap_target.output.mapfile +check_station_mismatch.control.h5parmdb = {{ solutions }} +check_station_mismatch.control.solset_name = calibrator +check_station_mismatch.control.filter = {{ filter_baselines }} + + +################################### +## Prepare for demixing/clipping ## +################################### +# generate a mapfile of the target +createmap_preptarg.control.kind = plugin +createmap_preptarg.control.type = makeResultsMapfile +createmap_preptarg.control.mapfile_dir = {{ mapfile_dir }} +createmap_preptarg.control.filename = createmap_preptarg.mapfile +createmap_preptarg.control.mapfile_in = createmap_target.output.mapfile +createmap_preptarg.control.target_dir = {{ working_directory }} +createmap_preptarg.control.make_target_dir = False +createmap_preptarg.control.new_suffix = .ndppp_prep_target + +# generate a mapfile for the instrument table of the target +createmap_insttarg.control.kind = plugin +createmap_insttarg.control.type = changeMapfile +createmap_insttarg.control.mapfile_in = createmap_preptarg.output.mapfile +createmap_insttarg.control.join_files = instrument +createmap_insttarg.control.newname = createmap_insttarg.mapfile + +# create a mapfile with the A-Team skymodel, length = 1 +create_ateam_model_map.control.kind = plugin +create_ateam_model_map.control.type = addListMapfile +create_ateam_model_map.control.hosts = ['localhost'] +create_ateam_model_map.control.files = [ {{ A-team_skymodel }} ] +create_ateam_model_map.control.mapfile_dir = {{ mapfile_dir }} +create_ateam_model_map.control.filename = ateam_model_name.mapfile + +# make sourcedbs from the A-Team skymodel, length = 1 +make_sourcedb_ateam.control.kind = recipe +make_sourcedb_ateam.control.type = executable_args +make_sourcedb_ateam.control.executable = {{ lofar_directory }}/bin/makesourcedb +make_sourcedb_ateam.control.error_tolerance = {{ error_tolerance }} +make_sourcedb_ateam.control.args_format = lofar +make_sourcedb_ateam.control.outputkey = out +make_sourcedb_ateam.control.mapfile_in = create_ateam_model_map.output.mapfile +make_sourcedb_ateam.control.inputkey = in +make_sourcedb_ateam.argument.format = < +make_sourcedb_ateam.argument.outtype = blob + +# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles +expand_sourcedb_ateam.control.kind = plugin +expand_sourcedb_ateam.control.type = expandMapfile +expand_sourcedb_ateam.control.mapfile_in = make_sourcedb_ateam.output.mapfile +expand_sourcedb_ateam.control.mapfile_to_match = createmap_target.output.mapfile +expand_sourcedb_ateam.control.mapfile_dir = {{ mapfile_dir }} +expand_sourcedb_ateam.control.filename = expand_sourcedb_ateam.datamap + + +############################# +## RM target correction ## +############################# +# get ionex files once for every day that is covered by one of the input MSs +h5imp_RMextract.control.type = pythonplugin +h5imp_RMextract.control.executable = {{ scripts }}/createRMh5parm.py +h5imp_RMextract.control.error_tolerance = {{ error_tolerance }} +h5imp_RMextract.argument.flags = [combine_data_target_map.output.mapfile, {{ solutions }}] +h5imp_RMextract.argument.ionex_server = {{ ionex_server }} +h5imp_RMextract.argument.ionex_prefix = {{ ionex_prefix }} +h5imp_RMextract.argument.ionexPath = {{ ionex_path }} +h5imp_RMextract.argument.solset_name = target +h5imp_RMextract.argument.proxyServer = {{ proxy_server }} +h5imp_RMextract.argument.proxyPort = {{ proxy_port }} +h5imp_RMextract.argument.proxyType = {{ proxy_type }} +h5imp_RMextract.argument.proxyUser = {{ proxy_user }} +h5imp_RMextract.argument.proxyPass = {{ proxy_pass }} + +# create losoto v2 parset file +prepare_losoto_RMextract.control.kind = plugin +prepare_losoto_RMextract.control.type = makeLosotoParset +prepare_losoto_RMextract.control.steps = [plotRM] +prepare_losoto_RMextract.control.filename = {{ job_directory }}/losoto.parset +prepare_losoto_RMextract.control.global.ncpu = {{ num_proc_per_node }} +prepare_losoto_RMextract.control.plotRM.operation = PLOT +prepare_losoto_RMextract.control.plotRM.soltab = target/RMextract +prepare_losoto_RMextract.control.plotRM.axesInPlot = time +prepare_losoto_RMextract.control.plotRM.axisInTable = ant +prepare_losoto_RMextract.control.plotRM.prefix = {{ inspection_directory }}/RMextract + +# do the processing on the LoSoTo file +process_losoto_RMextract.control.kind = recipe +process_losoto_RMextract.control.type = executable_args +process_losoto_RMextract.control.executable = {{ losoto_directory }}/bin/losoto +process_losoto_RMextract.control.max_per_node = {{ num_proc_per_node }} +process_losoto_RMextract.control.mapfile_in = combine_data_target_map.output.mapfile +process_losoto_RMextract.control.inputkey = input +process_losoto_RMextract.argument.flags = [{{ solutions }}, {{ job_directory }}/losoto.parset] + + +############################# +## Apply calibrator sols ## +############################# +# run NDPPP on the target data to flag, transfer calibrator values, and average +ndppp_prep_target.control.type = dppp +ndppp_prep_target.control.max_per_node = {{ num_proc_per_node_limit }} +ndppp_prep_target.control.error_tolerance = {{ error_tolerance }} +ndppp_prep_target.control.mapfiles_in = [createmap_target.output.mapfile,expand_sourcedb_ateam.output.mapfile,createmap_insttarg.output.mapfile] +ndppp_prep_target.control.inputkeys = [input_file,sourcedb,instrument] +ndppp_prep_target.argument.numthreads = {{ max_dppp_threads }} +ndppp_prep_target.argument.msin = input_file +ndppp_prep_target.argument.msin.datacolumn = DATA +ndppp_prep_target.argument.msin.baseline = check_station_mismatch.output.filter +ndppp_prep_target.argument.msin.autoweight = {{ raw_data }} +ndppp_prep_target.argument.msout.datacolumn = DATA +ndppp_prep_target.argument.msout.writefullresflag = False +ndppp_prep_target.argument.msout.overwrite = True +ndppp_prep_target.argument.msout.storagemanager = "Dysco" +ndppp_prep_target.argument.msout.storagemanager.databitrate = 0 +ndppp_prep_target.argument.steps = [{{ initial_flagging }},{{ demix_step }}filter,applyPA,applybandpass,{{ apply_steps }},avg] +ndppp_prep_target.argument.filter.type = filter +ndppp_prep_target.argument.filter.baseline = check_station_mismatch.output.filter +ndppp_prep_target.argument.filter.remove = true +ndppp_prep_target.argument.flagedge.type = preflagger +ndppp_prep_target.argument.flagedge.chan = [0..nchan/32-1,31*nchan/32..nchan-1] # we are running on a single subband +ndppp_prep_target.argument.aoflag.type = aoflagger +ndppp_prep_target.argument.aoflag.memoryperc = 10 +ndppp_prep_target.argument.aoflag.keepstatistics = false +ndppp_prep_target.argument.flagbaseline.type = preflagger +ndppp_prep_target.argument.flagbaseline.baseline = {{ flag_baselines }} +ndppp_prep_target.argument.flagelev.type = preflagger +ndppp_prep_target.argument.flagelev.elevation = 0deg..20deg +ndppp_prep_target.argument.flagamp.type = preflagger +ndppp_prep_target.argument.flagamp.amplmin = 1e-30 +ndppp_prep_target.argument.applyPA.type = applycal +ndppp_prep_target.argument.applyPA.parmdb = {{ solutions }} +ndppp_prep_target.argument.applyPA.correction = polalign +ndppp_prep_target.argument.applyPA.solset = calibrator +ndppp_prep_target.argument.applybandpass.type = applycal +ndppp_prep_target.argument.applybandpass.parmdb = {{ solutions }} +ndppp_prep_target.argument.applybandpass.correction = bandpass +ndppp_prep_target.argument.applybandpass.updateweights = {{ updateweights }} +ndppp_prep_target.argument.applybandpass.solset = calibrator +ndppp_prep_target.argument.applyclock.type = applycal +ndppp_prep_target.argument.applyclock.parmdb = {{ solutions }} +ndppp_prep_target.argument.applyclock.correction = clock +ndppp_prep_target.argument.applyclock.solset = calibrator +ndppp_prep_target.argument.applytec.type = applycal +ndppp_prep_target.argument.applytec.parmdb = {{ solutions }} +ndppp_prep_target.argument.applytec.correction = tec +ndppp_prep_target.argument.applytec.solset = calibrator +ndppp_prep_target.argument.applyphase.type = applycal +ndppp_prep_target.argument.applyphase.parmdb = {{ solutions }} +ndppp_prep_target.argument.applyphase.correction = phaseOrig +ndppp_prep_target.argument.applyphase.solset = calibrator +ndppp_prep_target.argument.applyRM.type = applycal +ndppp_prep_target.argument.applyRM.parmdb = {{ solutions }} +ndppp_prep_target.argument.applyRM.correction = RMextract +ndppp_prep_target.argument.applyRM.solset = target +ndppp_prep_target.argument.applybeam.type = applybeam +ndppp_prep_target.argument.applybeam.usechannelfreq = True +ndppp_prep_target.argument.applybeam.updateweights = {{ updateweights }} +ndppp_prep_target.argument.applybeam.invert = True +ndppp_prep_target.argument.avg.type = average +ndppp_prep_target.argument.avg.timeresolution = {{ avg_timeresolution }} +ndppp_prep_target.argument.avg.freqresolution = {{ avg_freqresolution }} +ndppp_prep_target.argument.demix.type = demixer +ndppp_prep_target.argument.demix.baseline = {{ process_baselines_target }} +ndppp_prep_target.argument.demix.demixfreqstep = {{ demix_freqstep }} +ndppp_prep_target.argument.demix.demixtimestep = {{ demix_timestep }} +ndppp_prep_target.argument.demix.ignoretarget = False +ndppp_prep_target.argument.demix.targetsource = {{ demix_target }} +ndppp_prep_target.argument.demix.subtractsources = {{ demix_sources }} +ndppp_prep_target.argument.demix.ntimechunk = {{ max_dppp_threads }} +ndppp_prep_target.argument.demix.skymodel = sourcedb +ndppp_prep_target.argument.demix.freqstep = 1 +ndppp_prep_target.argument.demix.timestep = 1 +ndppp_prep_target.argument.demix.instrumentmodel = instrument + + +############################# +## Clip A-Team ## +############################# +# Predict, corrupt, and predict the ateam-resolution model, length = nfiles +predict_ateam.control.type = dppp +predict_ateam.control.mapfiles_in = [ndppp_prep_target.output.mapfile,expand_sourcedb_ateam.output.mapfile] +predict_ateam.control.inputkeys = [msin,sourcedb] +predict_ateam.control.inplace = True +predict_ateam.control.max_per_node = {{ num_proc_per_node_limit }} +predict_ateam.control.error_tolerance = {{ error_tolerance }} +predict_ateam.argument.numthreads = {{ max_dppp_threads }} +predict_ateam.argument.msin.datacolumn = DATA +predict_ateam.argument.msout.datacolumn = MODEL_DATA +predict_ateam.argument.msout.storagemanager = "Dysco" +predict_ateam.argument.msout.storagemanager.databitrate = 0 +predict_ateam.argument.steps = [filter,predict] +predict_ateam.argument.filter.type = filter +predict_ateam.argument.filter.baseline = {{ process_baselines_target }} +predict_ateam.argument.filter.remove = False +predict_ateam.argument.predict.type = predict +predict_ateam.argument.predict.operation = replace +predict_ateam.argument.predict.sourcedb = sourcedb +predict_ateam.argument.predict.sources = [VirA_4_patch,CygAGG,CasA_4_patch,TauAGG] +predict_ateam.argument.predict.usebeammodel = True +predict_ateam.argument.predict.usechannelfreq = False +predict_ateam.argument.predict.onebeamperpatch = True + +# run the a-team clipper to flag data affected by the a-team +ateamcliptar.control.kind = recipe +ateamcliptar.control.type = executable_args +ateamcliptar.control.max_per_node = {{ num_proc_per_node }} +ateamcliptar.control.executable = {{ scripts }}/Ateamclipper.py +ateamcliptar.control.error_tolerance = {{ error_tolerance }} +ateamcliptar.control.mapfile_in = ndppp_prep_target.output.mapfile +ateamcliptar.control.arguments = [allms] +ateamcliptar.control.inputkey = allms + +# run the a-team clipper to flag data affected by the a-team +plotateamclip.control.type = pythonplugin +plotateamclip.control.executable = {{ scripts }}/plot_Ateamclipper.py +plotateamclip.control.error_tolerance = {{ error_tolerance }} +plotateamclip.control.skip_infile = True +plotateamclip.control.mapfile_in = combine_data_target_map.output.mapfile +plotateamclip.argument.txtfile = {{ working_directory }}/Ateamclipper.txt +plotateamclip.argument.outfile = {{ inspection_directory }}/Ateamclipper.png + +############################# +## concatenate ## +############################# +# combine all entries into one mapfile, for the sortmap script +combine_target_map.control.kind = plugin +combine_target_map.control.type = createMapfile +combine_target_map.control.method = mapfile_all_to_one +combine_target_map.control.mapfile_dir = {{ mapfile_dir }} +combine_target_map.control.filename = combine_target_map.mapfile +combine_target_map.control.mapfile_in = ndppp_prep_target.output.mapfile + +# check bad antennas +check_bad_antennas.control.kind = plugin +check_bad_antennas.control.type = identifyBadAntennas +check_bad_antennas.control.mapfile_in = ndppp_prep_target.output.mapfile +check_bad_antennas.control.filter = {{ process_baselines_target }} + +# sort the target data by frequency into groups so that NDPPP can concatenate them +sortmap_target.control.type = pythonplugin +sortmap_target.control.executable = {{ scripts }}/sort_times_into_freqGroups.py +sortmap_target.argument.flags = [combine_target_map.output.mapfile] +sortmap_target.argument.filename = sortmap_target +sortmap_target.argument.mapfile_dir = {{ mapfile_dir }} +sortmap_target.argument.target_path = {{ working_directory }} +sortmap_target.argument.numSB = {{ num_SBs_per_group }} +sortmap_target.argument.NDPPPfill = True +sortmap_target.argument.stepname = dpppconcat +sortmap_target.argument.firstSB = {{ reference_stationSB }} +sortmap_target.argument.truncateLastSBs = False + +# convert the output of sortmap_target into usable mapfiles +do_sortmap_maps.control.kind = plugin +do_sortmap_maps.control.type = mapfilenamesFromMapfiles +do_sortmap_maps.control.mapfile_groupmap = sortmap_target.output.groupmapfile.mapfile +do_sortmap_maps.control.mapfile_datamap = sortmap_target.output.mapfile.mapfile + +# run NDPPP to concatenate the target +dpppconcat.control.type = dppp +dpppconcat.control.max_per_node = {{ num_proc_per_node_limit }} +dpppconcat.control.error_tolerance = {{ error_tolerance }} +dpppconcat.control.mapfile_out = do_sortmap_maps.output.groupmap # tell the pipeline to give the output useful names +dpppconcat.control.mapfiles_in = [do_sortmap_maps.output.datamap] +dpppconcat.control.inputkey = msin +dpppconcat.argument.msin.datacolumn = DATA +dpppconcat.argument.msin.missingdata = True #\ these two lines will make NDPPP generate dummy data when +dpppconcat.argument.msin.orderms = False #/ concatenating data +dpppconcat.argument.msin.baseline = check_bad_antennas.output.filter +dpppconcat.argument.filter.type = filter +dpppconcat.argument.filter.baseline = check_bad_antennas.output.filter +dpppconcat.argument.filter.remove = True +dpppconcat.argument.msout.datacolumn = DATA +dpppconcat.argument.msout.writefullresflag = False +dpppconcat.argument.msout.overwrite = True +dpppconcat.argument.msout.storagemanager = "Dysco" +dpppconcat.argument.msout.storagemanager.databitrate = 0 +dpppconcat.argument.steps = [filter,avg] +dpppconcat.argument.avg.type = average +dpppconcat.argument.avg.timeresolution = {{ avg_timeresolution_concat }} +dpppconcat.argument.avg.freqresolution = {{ avg_freqresolution_concat }} + +# combine all entries into one mapfile, for the sortmap script +combine_concat_map.control.kind = plugin +combine_concat_map.control.type = createMapfile +combine_concat_map.control.method = mapfile_all_to_one +combine_concat_map.control.mapfile_dir = {{ mapfile_dir }} +combine_concat_map.control.filename = combine_concat_map.mapfile +combine_concat_map.control.mapfile_in = do_sortmap_maps.output.groupmap + +# virtually concatenate target subbands +ms_concat_target.control.type = pythonplugin +ms_concat_target.control.executable = {{ scripts }}/concat_MS.py +ms_concat_target.control.error_tolerance = {{ error_tolerance }} +ms_concat_target.argument.filename = concatmapfile.mapfile +ms_concat_target.argument.mapfile_dir = {{ mapfile_dir }} +ms_concat_target.argument.min_length = {{ min_length }} +ms_concat_target.argument.overhead = {{ overhead }} +ms_concat_target.argument.flags = [combine_concat_map.output.mapfile,outputkey] + +# convert the output of ms_concat_target into usable mapfiles +ms_concat_target_map.control.kind = plugin +ms_concat_target_map.control.type = mapfilenamesFromMapfiles +ms_concat_target_map.control.mapfile_concatmap = ms_concat_target.output.concatmapfile.mapfile + +# convert the output of ms_concat_target into usable mapfiles +expand_memory_map.control.kind = plugin +expand_memory_map.control.type = expandMapfile +expand_memory_map.control.mapfile_in = ms_concat_target.output.memory.mapfile +expand_memory_map.control.mapfile_to_match = ms_concat_target_map.output.concatmap +expand_memory_map.control.mapfile_dir = {{ mapfile_dir }} +expand_memory_map.control.filename = expand_memory_map.mapfile + +# run aoflagger on the concatenated data +aoflag.control.kind = recipe +aoflag.control.type = executable_args +aoflag.control.inplace = True +aoflag.control.executable = {{ aoflagger }} +aoflag.control.max_per_node = 1 +aoflag.control.error_tolerance = {{ error_tolerance }} +aoflag.control.mapfiles_in = [ms_concat_target_map.output.concatmap,expand_memory_map.output.mapfile] +aoflag.control.inputkeys = [msin,memory] +aoflag.control.args_format = wsclean +aoflag.argument.strategy = {{ prefactor_directory }}/rfistrategies/{{ rfistrategy }} +aoflag.argument.flags = [-v,memory,-combine-spws,msin] + + +############################# +## phasecal target ## +############################# +#check all files for minimum unflagged fraction +check_unflagged.control.type = pythonplugin +check_unflagged.control.executable = {{ scripts }}/check_unflagged_fraction.py +check_unflagged.argument.flags = [dpppconcat.output.mapfile] +check_unflagged.argument.min_fraction = {{ min_unflagged_fraction }} + +# prune flagged files from mapfile +check_unflagged_map.control.kind = plugin +check_unflagged_map.control.type = pruneMapfile +check_unflagged_map.control.mapfile_in = check_unflagged.output.flagged.mapfile +check_unflagged_map.control.mapfile_dir = {{ mapfile_dir }} +check_unflagged_map.control.filename = check_unflagged_map.mapfile +check_unflagged_map.control.prune_str = None + +# compress mapfiles for plotting +combine_concat_map.control.kind = plugin +combine_concat_map.control.type = compressMapfile +combine_concat_map.control.mapfile_in = dpppconcat.output.mapfile +combine_concat_map.control.mapfile_dir = {{ mapfile_dir }} +combine_concat_map.control.filename = combine_concat_map.mapfile + +# compress mapfiles for plotting +combine_frac_map.control.kind = plugin +combine_frac_map.control.type = compressMapfile +combine_frac_map.control.mapfile_in = check_unflagged.output.unflagged_fraction.mapfile +combine_frac_map.control.mapfile_dir = {{ mapfile_dir }} +combine_frac_map.control.filename = combine_frac_map.mapfile + +# plot the unflagged fraction +plot_unflagged.control.type = pythonplugin +plot_unflagged.control.executable = {{ scripts }}/plot_unflagged_fraction.py +plot_unflagged.control.mapfiles_in = [combine_concat_map.output.mapfile,combine_frac_map.output.mapfile] +plot_unflagged.control.inputkeys = [msin,frac] +plot_unflagged.argument.flags = [msin,frac] +plot_unflagged.argument.outfile = {{ inspection_directory }}/unflagged_fraction.png + +# if wished, download the tgss skymodel for the target +sky_tar.control.type = pythonplugin +sky_tar.control.executable = {{ scripts }}/download_skymodel_target.py +sky_tar.argument.flags = [combine_target_map.output.mapfile] +sky_tar.argument.DoDownload = {{ use_target }} +sky_tar.argument.SkymodelPath = {{ target_skymodel }} +sky_tar.argument.Radius = 5. #in degrees +sky_tar.argument.Source = {{ skymodel_source }} + +# create a mapfile with the target skymodel, length = 1 +create_target_model_map.control.kind = plugin +create_target_model_map.control.type = addListMapfile +create_target_model_map.control.hosts = ['localhost'] +create_target_model_map.control.files = [ {{ target_skymodel }} ] +create_target_model_map.control.mapfile_dir = {{ mapfile_dir }} +create_target_model_map.control.filename = target_model_name.mapfile + +# make sourcedbs from the target skymodel, length = 1 +make_sourcedb_target.control.kind = recipe +make_sourcedb_target.control.type = executable_args +make_sourcedb_target.control.executable = {{ lofar_directory }}/bin/makesourcedb +make_sourcedb_target.control.error_tolerance = {{ error_tolerance }} +make_sourcedb_target.control.args_format = lofar +make_sourcedb_target.control.outputkey = out +make_sourcedb_target.control.mapfile_in = create_target_model_map.output.mapfile +make_sourcedb_target.control.inputkey = in +make_sourcedb_target.argument.format = < +make_sourcedb_target.argument.outtype = blob + +# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles +expand_sourcedb_target.control.kind = plugin +expand_sourcedb_target.control.type = expandMapfile +expand_sourcedb_target.control.mapfile_in = make_sourcedb_target.output.mapfile +expand_sourcedb_target.control.mapfile_to_match = check_unflagged_map.output.mapfile +expand_sourcedb_target.control.mapfile_dir = {{ mapfile_dir }} +expand_sourcedb_target.control.filename = expand_sourcedb_target.datamap + +# generate mapfile with the parmDB names to be used in the gsmcal steps +gsmcal_parmmap.control.kind = plugin +gsmcal_parmmap.control.type = createMapfile +gsmcal_parmmap.control.method = add_suffix_to_file +gsmcal_parmmap.control.mapfile_in = check_unflagged_map.output.mapfile +gsmcal_parmmap.control.add_suffix_to_file = .h5 +gsmcal_parmmap.control.mapfile_dir = {{ mapfile_dir }} +gsmcal_parmmap.control.filename = gsmcal_parmdbs.mapfile + +# generate a mapfile with all files in a single entry +h5_gsmsol_map.control.kind = plugin +h5_gsmsol_map.control.type = compressMapfile +h5_gsmsol_map.control.mapfile_in = gsmcal_parmmap.output.mapfile +h5_gsmsol_map.control.mapfile_dir = {{ mapfile_dir }} +h5_gsmsol_map.control.filename = h5_imp_gsmsol_map.mapfile + +# baseline-dependent smoothing +smooth_data.control.type = executable_args +smooth_data.control.inplace = True +smooth_data.control.max_per_node = {{ num_proc_per_node }} +smooth_data.control.error_tolerance = {{ error_tolerance }} +smooth_data.control.executable = {{ scripts }}/BLsmooth.py +smooth_data.control.mapfile_in = check_unflagged_map.output.mapfile +smooth_data.control.inputkey = msin +smooth_data.argument.flags = [-S,{{ do_smooth }},-r,-f,0.2,-i,DATA,-o,SMOOTHED_DATA,msin] + +# solve/store direction-independent phase-only self-calibration corrected UV-data to a fully Dysco compressed new MS +gsmcal_phase.control.type = dppp +gsmcal_phase.control.error_tolerance = {{ error_tolerance }} +gsmcal_phase.control.inplace = True +gsmcal_phase.control.max_per_node = {{ num_proc_per_node_limit }} +gsmcal_phase.control.mapfiles_in = [check_unflagged_map.output.mapfile,expand_sourcedb_target.output.mapfile,gsmcal_parmmap.output.mapfile] +gsmcal_phase.control.inputkeys = [input_file,sourcedb,parmdb] +gsmcal_phase.argument.msin = input_file +gsmcal_phase.argument.msin.datacolumn = SMOOTHED_DATA +gsmcal_phase.argument.numthreads = {{ max_dppp_threads }} +gsmcal_phase.argument.steps = [filter,gaincal] +gsmcal_phase.argument.filter.type = filter +gsmcal_phase.argument.filter.blrange = [150, 999999] +gsmcal_phase.argument.gaincal.type = gaincal +gsmcal_phase.argument.gaincal.parmdb = parmdb +gsmcal_phase.argument.gaincal.caltype = phaseonly +gsmcal_phase.argument.gaincal.sourcedb = sourcedb +gsmcal_phase.argument.gaincal.maxiter = 50 +gsmcal_phase.argument.gaincal.solint = 1 +gsmcal_phase.argument.gaincal.nchan = 0 +gsmcal_phase.argument.gaincal.tolerance = 1e-3 +gsmcal_phase.argument.gaincal.propagatesolutions = {{ propagatesolutions }} +gsmcal_phase.argument.gaincal.usebeammodel = True +gsmcal_phase.argument.gaincal.usechannelfreq = True +gsmcal_phase.argument.gaincal.beammode = array_factor +gsmcal_phase.argument.gaincal.onebeamperpatch = False + +# solve for direction-independent TEC +gsmcal_tec.control.type = dppp +gsmcal_tec.control.error_tolerance = {{ error_tolerance }} +gsmcal_tec.control.inplace = True +gsmcal_tec.control.max_per_node = {{ num_proc_per_node_limit }} +gsmcal_tec.control.mapfiles_in = [check_unflagged_map.output.mapfile,expand_sourcedb_target.output.mapfile,gsmcal_parmmap.output.mapfile] +gsmcal_tec.control.inputkeys = [input_file,sourcedb,parmdb] +gsmcal_tec.argument.msin = input_file +gsmcal_tec.argument.numthreads = {{ num_proc_per_node }} +gsmcal_tec.argument.msin.datacolumn = SMOOTHED_DATA +gsmcal_tec.argument.steps = [teccal] +gsmcal_tec.argument.teccal.type = ddecal +gsmcal_tec.argument.teccal.mode = tec +gsmcal_tec.argument.teccal.h5parm = parmdb +gsmcal_tec.argument.teccal.sourcedb = sourcedb +gsmcal_tec.argument.teccal.uvlambdamin = 100 +gsmcal_tec.argument.teccal.maxiter = 400 +gsmcal_tec.argument.teccal.solint = 3 +gsmcal_tec.argument.teccal.nchan = 8 +gsmcal_tec.argument.teccal.tolerance = 1e-3 +gsmcal_tec.argument.teccal.stepsize = 0.2 +gsmcal_tec.argument.teccal.approximatetec = True +gsmcal_tec.argument.teccal.maxapproxiter = 400 +gsmcal_tec.argument.teccal.approxtolerance = 1e-3 +gsmcal_tec.argument.teccal.propagatesolutions = {{ propagatesolutions }} +gsmcal_tec.argument.teccal.usebeammodel = True +gsmcal_tec.argument.teccal.usechannelfreq = True +gsmcal_tec.argument.teccal.beammode = array_factor +gsmcal_tec.argument.teccal.onebeamperpatch = False + + +########################### +## Analyze cal ## +########################### +# collect all instrument tables into one h5parm +h5imp_gsmcal.control.kind = recipe +h5imp_gsmcal.control.type = executable_args +h5imp_gsmcal.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py +h5imp_gsmcal.control.error_tolerance = {{ error_tolerance }} +h5imp_gsmcal.control.mapfile_in = h5_gsmsol_map.output.mapfile +h5imp_gsmcal.control.inputkey = h5in +h5imp_gsmcal.control.outputkey = outh5parm +h5imp_gsmcal.argument.flags = [-q,-v,-c,h5in] +h5imp_gsmcal.argument.outh5parm = outh5parm + +# create losoto v2 parset file +prepare_losoto_phase.control.kind = plugin +prepare_losoto_phase.control.type = makeLosotoParset +prepare_losoto_phase.control.steps = [plotP, plotP2, plotPd, plotPd2] +prepare_losoto_phase.control.filename = {{ job_directory }}/losoto.parset +prepare_losoto_phase.control.global.ncpu = {{ num_proc_per_node }} +prepare_losoto_phase.control.plotP.operation = PLOT +prepare_losoto_phase.control.plotP.soltab = sol000/phase000 +prepare_losoto_phase.control.plotP.axesInPlot = [time,freq] +prepare_losoto_phase.control.plotP.axisInTable = ant +prepare_losoto_phase.control.plotP.plotFlag = True +prepare_losoto_phase.control.plotP.prefix = {{ inspection_directory }}/ph_ +prepare_losoto_phase.control.plotP.refAnt = {{ refant }} +prepare_losoto_phase.control.plotP.minmax = [-3.14,3.14] +prepare_losoto_phase.control.plotP2.operation = PLOT +prepare_losoto_phase.control.plotP2.soltab = sol000/phase000 +prepare_losoto_phase.control.plotP2.axesInPlot = [time] +prepare_losoto_phase.control.plotP2.axisInTable = ant +prepare_losoto_phase.control.plotP2.axisInCol = pol +prepare_losoto_phase.control.plotP2.plotFlag = True +prepare_losoto_phase.control.plotP2.prefix = {{ inspection_directory }}/ph_ +prepare_losoto_phase.control.plotP2.refAnt = {{ refant }} +prepare_losoto_phase.control.plotP2.minmax = [-3.14,3.14] +prepare_losoto_phase.control.plotPd.operation = PLOT +prepare_losoto_phase.control.plotPd.soltab = sol000/phase000 +prepare_losoto_phase.control.plotPd.axesInPlot = [time,freq] +prepare_losoto_phase.control.plotPd.axisInTable = ant +prepare_losoto_phase.control.plotPd.axisDiff = pol +prepare_losoto_phase.control.plotPd.plotFlag = True +prepare_losoto_phase.control.plotPd.prefix = {{ inspection_directory }}/ph_poldif +prepare_losoto_phase.control.plotPd.refAnt = {{ refant }} +prepare_losoto_phase.control.plotPd.minmax = [-3.14,3.14] +prepare_losoto_phase.control.plotPd2.operation = PLOT +prepare_losoto_phase.control.plotPd2.soltab = sol000/phase000 +prepare_losoto_phase.control.plotPd2.axesInPlot = [time] +prepare_losoto_phase.control.plotPd2.axisInTable = ant +prepare_losoto_phase.control.plotPd2.axisDiff = pol +prepare_losoto_phase.control.plotPd2.plotFlag = True +prepare_losoto_phase.control.plotPd2.prefix = {{ inspection_directory }}/ph_poldif_ +prepare_losoto_phase.control.plotPd2.refAnt = {{ refant }} +prepare_losoto_phase.control.plotPd2.minmax = [-3.14,3.14] + +# create losoto v2 parset file +prepare_losoto_tec.control.kind = plugin +prepare_losoto_tec.control.type = makeLosotoParset +prepare_losoto_tec.control.steps = [duplicatePbkp,plotTEC1,dejump,plotTEC2] +prepare_losoto_tec.control.filename = {{ job_directory }}/losoto.parset +prepare_losoto_tec.control.global.ncpu = {{ num_proc_per_node }} +prepare_losoto_tec.control.duplicatePbkp.operation = DUPLICATE +prepare_losoto_tec.control.duplicatePbkp.soltab = sol000/tec000 +prepare_losoto_tec.control.duplicatePbkp.soltabOut = tecOrig000 +prepare_losoto_tec.control.plotTEC1.operation = PLOT +prepare_losoto_tec.control.plotTEC1.soltab = sol000/tec000 +prepare_losoto_tec.control.plotTEC1.axesInPlot = time +prepare_losoto_tec.control.plotTEC1.axisInTable = ant +prepare_losoto_tec.control.plotTEC1.plotFlag = True +prepare_losoto_tec.control.plotTEC1.minmax = [-0.5,0.5] +prepare_losoto_tec.control.plotTEC1.prefix = {{ inspection_directory }}/tec +prepare_losoto_tec.control.plotTEC1.refAnt = {{ refant }} +prepare_losoto_tec.control.dejump.operation = TECJUMP +prepare_losoto_tec.control.dejump.soltab = sol000/tec000 +prepare_losoto_tec.control.dejump.refAnt = {{ refant }} +prepare_losoto_tec.control.plotTEC2.operation = PLOT +prepare_losoto_tec.control.plotTEC2.soltab = sol000/tec000 +prepare_losoto_tec.control.plotTEC2.axesInPlot = time +prepare_losoto_tec.control.plotTEC2.axisInTable = ant +prepare_losoto_tec.control.plotTEC2.plotFlag = True +prepare_losoto_tec.control.plotTEC2.minmax = [-0.5,0.5] +prepare_losoto_tec.control.plotTEC2.prefix = {{ inspection_directory }}/tec_nojump +prepare_losoto_tec.control.plotTEC2.refAnt = {{ refant }} + +# do the processing on the LoSoTo file +process_losoto_gsmcal.control.kind = recipe +process_losoto_gsmcal.control.type = executable_args +process_losoto_gsmcal.control.inplace = True +process_losoto_gsmcal.control.executable = {{ losoto_directory }}/bin/losoto +process_losoto_gsmcal.control.max_per_node = {{ num_proc_per_node }} +process_losoto_gsmcal.control.mapfile_in = h5imp_gsmcal.output.mapfile +process_losoto_gsmcal.control.inputkey = h5in +process_losoto_gsmcal.argument.flags = [h5in,{{ job_directory }}/losoto.parset] + +# add missing stations to the soltab if any +add_missing_stations.control.type = pythonplugin +add_missing_stations.control.inplace = True +add_missing_stations.control.executable = {{ scripts }}/add_missing_stations.py +add_missing_stations.control.error_tolerance = {{ error_tolerance }} +add_missing_stations.control.mapfile_in = h5imp_gsmcal.output.mapfile +add_missing_stations.control.inputkey = h5in +add_missing_stations.argument.flags = [h5in] +add_missing_stations.argument.solset = sol000 +add_missing_stations.argument.refsolset = target +add_missing_stations.argument.refh5 = {{ solutions }} +add_missing_stations.argument.soltab_in = {{ gsmcal_step }}000 +add_missing_stations.argument.soltab_out = {{ skymodel_source }}{{ gsmcal_step }} +add_missing_stations.argument.bad_antennas = check_bad_antennas.output.filter +add_missing_stations.argument.filter = {{ process_baselines_target }} + +# output the final soltab into an external h5parm +h5exp_gsm.control.kind = recipe +h5exp_gsm.control.type = executable_args +h5exp_gsm.control.inplace = True +h5exp_gsm.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py +h5exp_gsm.control.error_tolerance = {{ error_tolerance }} +h5exp_gsm.control.mapfile_in = h5imp_gsmcal.output.mapfile +h5exp_gsm.control.inputkey = h5in +h5exp_gsm.argument.flags = [-q,-v,-H,h5in] +h5exp_gsm.argument.insoltab = {{ skymodel_source }}{{ gsmcal_step }} +h5exp_gsm.argument.outsolset = target +h5exp_gsm.argument.outh5parm = {{ solutions }} + +################################ +## final step (EXPERIMENTAL) ## +################################ + +# apply the final solutions to the data and compress it +apply_gsmcal.control.type = dppp +apply_gsmcal.control.error_tolerance = {{ error_tolerance }} +apply_gsmcal.control.max_per_node = {{ num_proc_per_node_limit }} +apply_gsmcal.argument.msin = check_unflagged_map.output.mapfile +apply_gsmcal.argument.numthreads = {{ max_dppp_threads }} +apply_gsmcal.argument.msin.datacolumn = DATA +apply_gsmcal.argument.msout.storagemanager = "Dysco" +apply_gsmcal.argument.msout.storagemanager.databitrate = {{ compression_bitrate }} +apply_gsmcal.argument.steps = [applygsm] +apply_gsmcal.argument.applygsm.type = applycal +apply_gsmcal.argument.applygsm.correction = {{ skymodel_source }}{{ gsmcal_step }} +apply_gsmcal.argument.applygsm.parmdb = {{ solutions }} +apply_gsmcal.argument.applygsm.solset = target + +# make mapfile with the filenames of the results that we want +make_results_mapfile.control.kind = plugin +make_results_mapfile.control.type = makeResultsMapfile +make_results_mapfile.control.mapfile_dir = {{ mapfile_dir }} +make_results_mapfile.control.filename = make_results_mapfile.mapfile +make_results_mapfile.control.mapfile_in = apply_gsmcal.output.mapfile +make_results_mapfile.control.target_dir = {{ results_directory }} +make_results_mapfile.control.make_target_dir = True +make_results_mapfile.control.new_suffix = .pre-cal.ms + +# compress mapfiles for plotting +make_results_compress.control.kind = plugin +make_results_compress.control.type = compressMapfile +make_results_compress.control.mapfile_in = make_results_mapfile.output.mapfile +make_results_compress.control.mapfile_dir = {{ mapfile_dir }} +make_results_compress.control.filename = make_results_compress.mapfile + +# move the results to where we want them +move_results.control.kind = recipe +move_results.control.type = executable_args +move_results.control.executable = /bin/mv +move_results.control.max_per_node = {{ num_proc_per_node_limit }} +move_results.control.mapfiles_in = [apply_gsmcal.output.mapfile,make_results_mapfile.output.mapfile] +move_results.control.inputkeys = [source,destination] +move_results.control.arguments = [source,destination] + +# set the pointing direction +h5parm_name.control.type = pythonplugin +h5parm_name.control.executable = {{ scripts }}/h5parm_pointingname.py +h5parm_name.control.error_tolerance = {{ error_tolerance }} +h5parm_name.control.skip_infile = True +h5parm_name.control.mapfile_in = combine_data_target_map.output.mapfile +h5parm_name.argument.flags = [{{ solutions }}] +h5parm_name.argument.solsetName = target +h5parm_name.argument.pointing = get_targetname.output.targetName + +# set the pointing direction +structure_function.control.type = pythonplugin +structure_function.control.executable = {{ scripts }}/getStructure_from_phases.py +structure_function.control.error_tolerance = {{ error_tolerance }} +structure_function.control.skip_infile = True +structure_function.control.mapfile_in = combine_data_target_map.output.mapfile +structure_function.argument.flags = [{{ solutions }}] +structure_function.argument.solset = target +structure_function.argument.soltab = {{ skymodel_source }}{{ gsmcal_step }} +structure_function.argument.outbasename = get_targetname.output.targetName +structure_function.argument.output_dir = {{ inspection_directory }} + +# set the pointing direction +make_summary.control.type = pythonplugin +make_summary.control.executable = {{ scripts }}/make_summary.py +make_summary.control.error_tolerance = {{ error_tolerance }} +make_summary.control.mapfile_in = make_results_compress.output.mapfile +make_summary.control.inputkey = infiles +make_summary.argument.observation_directory = {{ working_directory }} +make_summary.argument.logfile = {{ log_file }} +make_summary.argument.h5parmdb = {{ solutions }} +make_summary.argument.inspection_directory = {{ inspection_directory }} +make_summary.argument.MSfile = infiles + +######################################################## +## ## +## END PIPELINE ## +## ## +######################################################## \ No newline at end of file diff --git a/prefactor3/build_docker_container.sh b/prefactor3/build_docker_container.sh new file mode 100755 index 0000000000000000000000000000000000000000..d0e790a7046f9e5f356fb49ed1469b1ebd5fd0f6 --- /dev/null +++ b/prefactor3/build_docker_container.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker build . -t lofarit/prefactor3:pipeline_v3.10 diff --git a/prefactor3/data/input_data/README.md b/prefactor3/data/input_data/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2a76678e42d9602b4b20071b0f320e8b243e3548 --- /dev/null +++ b/prefactor3/data/input_data/README.md @@ -0,0 +1 @@ +Put here "calib" and "target" MS data (or link agains a folder containing them). diff --git a/prefactor3/data/output_data/README.md b/prefactor3/data/output_data/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6056688d43a08cbac0f8db741736fc57115028f0 --- /dev/null +++ b/prefactor3/data/output_data/README.md @@ -0,0 +1 @@ +Directory for output data. diff --git a/prefactor3/entrypoint.sh b/prefactor3/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..2f9574e71db74036f1f09dae94457c17f2216368 --- /dev/null +++ b/prefactor3/entrypoint.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Exit on any error. More complex thing could be done in future +# (see https://stackoverflow.com/questions/4381618/exit-a-script-on-error) +set -e + +echo "" +echo "[INFO] Executing entrypoint..." + +echo "[INFO] Sourcing env in /opt/lofarsoft/lofarinit.sh..." +source /opt/lofarsoft/lofarinit.sh + +echo "[INFO] Creating /tmp/lofarhome to be used as lofar home" +mkdir /tmp/lofarhome + +echo "[INFO] Initialising /tmp/lofarhome with configuration files" +cp -a /home/vanilla_lofar/* /tmp/lofarhome + +echo "[INFO] Moving to /home/lofar and setting as home" +cd /home/lofar +export HOME=/home/lofar + +echo "[INFO] Setting new prompt @prefactor3 container" +echo 'export PS1="${debian_chroot:+($debian_chroot)}\u@prefactor3@\h:\w\$ "' > /tmp/lofarhome/.bashrc + +# Set entrypoint command +if [ "x$@" == "x" ]; then + COMMAND="/bin/bash" +else + COMMAND="$@" +fi + + + +# Start! +echo -n "[INFO] Will execute entrypoint command: " +echo $COMMAND +echo "" +echo "==============================================================" +echo "| Welcome to the Prefactor3 container! |" +echo "==============================================================" +echo "" +echo "You are now in /home/lofar with write access as user \"$(whoami)\"." +echo "" +echo "Remember that contents inside this container, unless stored" +echo "on a persistent volume mounted from you host machine, will" +echo "be wiped out when exiting the container." +echo "" + +exec $COMMAND + diff --git a/prefactor3/pipeline.cfg b/prefactor3/pipeline.cfg new file mode 100644 index 0000000000000000000000000000000000000000..5ebd313994e5799aca84d4e372000b37c6b921a5 --- /dev/null +++ b/prefactor3/pipeline.cfg @@ -0,0 +1,44 @@ +[DEFAULT] +lofarroot = /opt/lofarsoft +casaroot = %(lofarroot)s +pyraproot = +hdf5root = +wcsroot = + +# /opt/lofar/lib/python2.7/site-packages +pythonpath = %(lofarroot)s/lib/python2.7/site-packages + +# /opt/lofar/lib/python2.7/site-packages/lofarpipe/recipes +recipe_directories = [%(pythonpath)s/lofarpipe/recipes] + +runtime_directory = /output_data +working_directory = %(runtime_directory)s + +# [/opt/lofar/share/pipeline/tasks.cfg] +task_files = [%(lofarroot)s/share/pipeline/tasks.cfg] + + +[layout] +job_directory = %(runtime_directory)s/%(job_name)s + +[cluster] +clusterdesc = %(working_directory)s/pipeline.clusterdesc + +[deploy] +engine_ppath = %(pythonpath)s:%(pyraproot)s/lib:/opt/cep/pythonlibs/lib/python/site-packages +engine_lpath = %(lofarroot)s/lib:%(casaroot)s/lib:%(pyraproot)s/lib:%(hdf5root)s/lib:%(wcsroot)s/lib + +[logging] +log_file = %(runtime_directory)s/%(job_name)s/logs/%(start_time)s/pipeline.log +xml_stat_file = %(runtime_directory)s/%(job_name)s/logs/%(start_time)s/statistics.xml + +[feedback] +# Method of providing feedback to LOFAR. +# Valid options: +# messagebus Send feedback and status using LCS/MessageBus +# none Do NOT send feedback and status +method = none + +[remote] +method = local +max_per_node = 1 diff --git a/prefactor3/run_docker_container.sh b/prefactor3/run_docker_container.sh new file mode 100755 index 0000000000000000000000000000000000000000..0bbbc6c620332d40fd065c9a5f123604bf8bdb93 --- /dev/null +++ b/prefactor3/run_docker_container.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker run --rm -v $PWD/data:/data -it lofarit/prefactor3:pipeline_v3.10 /bin/bash diff --git a/prefactor3/run_pipelines.sh b/prefactor3/run_pipelines.sh new file mode 100755 index 0000000000000000000000000000000000000000..e9c281d898bcd4f2aed9f734580d05efe5ac5cca --- /dev/null +++ b/prefactor3/run_pipelines.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +genericpipeline.py /home/lofar/Pre-Facet-Calibrator.parset -v -d -c /home/lofar/pipeline.cfg +genericpipeline.py /home/lofar/Pre-Facet-Calibrator.parset -v -d -c /home/lofar/pipeline.cfg \ No newline at end of file diff --git a/prefactor3/run_singularity_container.sh b/prefactor3/run_singularity_container.sh new file mode 100755 index 0000000000000000000000000000000000000000..97d698ed2b088f9fc058025ec95182b0e2e6ff35 --- /dev/null +++ b/prefactor3/run_singularity_container.sh @@ -0,0 +1,2 @@ +#!/bin/bash +singularity run --pid --writable-tmpfs --containall --cleanenv -Bdata/lofar/input_data:/data/lofar/input_data,data/lofar/output_data:/data/lofar/output_data docker://lofarit/prefactor3 \ No newline at end of file diff --git a/prefactor3/sudoers b/prefactor3/sudoers new file mode 100644 index 0000000000000000000000000000000000000000..47ab37c90fdec1df833409f825d2665fe7d1f899 --- /dev/null +++ b/prefactor3/sudoers @@ -0,0 +1,30 @@ +# +# This file MUST be edited with the 'visudo' command as root. +# +# Please consider adding local content in /etc/sudoers.d/ instead of +# directly modifying this file. +# +# See the man page for details on how to write a sudoers file. +# +Defaults env_reset +Defaults mail_badpass +Defaults secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +# Host alias specification + +# User alias specification + +# Cmnd alias specification + +# User privilege specification +root ALL=(ALL:ALL) ALL + +# Members of the admin group may gain root privileges +%admin ALL=(ALL) ALL + +# Allow members of group sudo to execute any command +%sudo ALL=(ALL:ALL) NOPASSWD:ALL + +# See sudoers(5) for more information on "#include" directives: + +#includedir /etc/sudoers.d