#! python
"""

Context : SRP
Module  : SRPREMPipelineManager.py
Author  : Stefano Covino
Date    : 02/02/2021
E-mail  : stefano.covino@brera.inaf.astro.it
URL:    : http://www.merate.mi.astro.it/utenti/covino
Purpose : Manage the execution of the REM pipeline

Usage   : SRPREMPipelineManager [-a] [-c] [-d] [-p] [-r] [-s arg1] [-v] [-z] 
            -a Astrometry computation
            -c Copy astrometrized frames to a given directory
            -d Database feeding
            -H Analysis of host data
            -k Keep files locally after processing
            -p Compute photometry
            -s Starting id (ross remir)
            -z Compute zero-point

History : (27/11/2010) First version.
        : (13/12/2010) Position of the selected objects.
        : (14/12/2010) Maximum number of backup files.
        : (20/12/2010) Managment of too small error circle for GRB counterpart.
        :               PROCSTAT and PHOTSTAB added.
        : (05/03/2011) Filter in GRB message.
        : (28/03/2011) AGN photometry for optical data too.
        : (25/06/2011) ZP for REMIR Z band too.
        : (11/09/2011) Better cosmetics.
        : (07/09/2012) Better altazimuthal positions and pointing information.
        : (05/04/2013) Internal temperature and focus position in database and more categories of souces analyzed.
        : (21/06/2013) Minor fix in documentation and management of wrong dates.
        : (24/06/2013) AUTO sextractor photometry.
        : (09/08/2013) More objects in astrometry computation.
        : (11/08/2013) Minor bugs solved.
        : (17/08/2013) Astrometry not for focus fields.
        : (18/08/2013) ROS2 photometry parameters.
        : (20/08/2013) Astrometry deepness based on observation delay.
        : (22/08/2013) Better management of catalogue query response.
        : (25/08/2013) Better management of delay computation.
        : (28/08/2013) Frame derotation center.
        : (29/08/2013) Minor improvements.
        : (30/08/2013) gzip copied frames.
        : (31/08/2013) Better ROS2 intregration.
        : (03/09/2013) Better standard star search.
        : (04/09/2013) Refined strategies for good pointing.
        : (08/09/2014) Message correction.
        : (17/12/2014) more control on catalogue quest errors.
        : (03/05/2016) M2 temperature for focus.
        : (10/05/2016) Better porting.
        : (13/03/2017) Python3 porting.
        : (16/05/2017) Minor update.
        : (18/05/2017) Minor update.
        : (04/02/2019) Filename in GRB counterpart message.
        : (02/02/2021) Betetr management of the astropy.log system.
"""


from optparse import OptionParser
import logging, logging.handlers, gzip, math, os, os.path, shutil, sys, time


from SRPPipelinesREM import REMPipelineManagerConstants
from SRPPipelines.CreateLocDB import CreateLocDB
from SRPPipelines.CreateRemDB import CreateRemDB
from SRPPipelines.IsRightHost import IsRightHost
from SRPPipelines.IsRightUser import IsRightUser
from SRPPipelinesREM.AstroDeepness import AstroDeepness
from SRPPipelinesREM.ComputeREMAltAz import ComputeREMAltAz
from SRPPipelinesREM.WhichREMImagingFrame import WhichREMImagingFrame
from SRPPipelinesREM.GetREMFrame import GetREMFrame
from SRPPipelinesREM.GetGRBInfo import GetGRBInfo
from SRPPipelinesREM.GetREMFrameInfo import GetREMFrameInfo
from SRPPipelinesREM.GetLCMeteoInfo import GetLCMeteoInfo
from SRPPipelinesREM.GetREMFocusData import GetREMFocusData
from SRPPipelinesREM.GetREMTemp import GetREMTemp
from SRPPipelinesREM.LoadProcessPointFName import LoadProcessPointFName
from SRPPipelinesREM.SaveProcessPointFName import SaveProcessPointFName
from SRPPipelinesREM.LoadREMTargets import LoadREMTargets
from SRPPipelines.SaveResult2LocDB import SaveResult2LocDB
from SRPPipelines.SaveResult2RemDB import SaveResult2RemDB
from SRP.SRPNet.SendEMail import SendEMail
from SRP.SRPSystem.Which import Which
from SRP.SRPSystem.Pipe import Pipe
from SRPFITS.Frames import EclipseConstants
from SRPFITS.Frames import SExtractorConstants
from SRPFITS.Fits.AddHeaderComment import AddHeaderComment
from SRPFITS.Fits.GetHeaderValue import GetHeaderValue
from astropy import log
log.setLevel('WARNING')


parser = OptionParser(usage="usage: %prog [-a] [-d] [-H] [-k] [-p] [-s arg1] [-v] [-z]", version="%prog 1.13.2")
parser.add_option("-a", "--astro", action="store_true", help="Astrometry computation")
parser.add_option("-c", "--copyastro", action="store", type="string", help="Copy astrometrized frames to a given directory")
parser.add_option("-d", "--database", action="store_true", help="Database feeding")
parser.add_option("-H", "--host", action="store_true", help="Analysis of host data")
parser.add_option("-k", "--keep", action="store_true", help="Keep files locally after processing")
parser.add_option("-p", "--photometry", action="store_true", help="Compute photometry")
parser.add_option("-s", "--startingid", action="store", nargs=2, type='int', help="Starting id (ross remir)")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="Fully describe operations")
parser.add_option("-z", "--zeropoint", action="store_true", help="Compute zero-point")
(options, args) = parser.parse_args()


# Check host or user
if not (IsRightHost(REMPipelineManagerConstants.AccHost) or IsRightUser(REMPipelineManagerConstants.AccUser)):
    parser.error("Only authorized host can run the pipeline.")


# Input safety checks
if options.zeropoint and not options.photometry:
    parser.error("Zero-point computation requires photometry.")


# Pipeline component safety check
if Which(REMPipelineManagerConstants.calend) == None:
    parser.error("%s not found." % REMPipelineManagerConstants.calend)
#
if options.photometry:
    if Which(REMPipelineManagerConstants.photo) == None:
        parser.error("%s not found." % REMPipelineManagerConstants.photo)
    elif Which(REMPipelineManagerConstants.sigmak) == None:
        parser.error("%s not found." % REMPipelineManagerConstants.sigmak)
    elif Which(REMPipelineManagerConstants.param) == None:
        parser.error("%s not found." % REMPipelineManagerConstants.param)
    elif Which(SExtractorConstants.SRPsex) == None and Which(SExtractorConstants.SRPsex_cyg) == None:
        parser.error("%s not found." % SExtractorConstants.SRPsex)
#
if options.zeropoint:
    if Which(REMPipelineManagerConstants.zpoint) == None:
        parser.error("%s not found." % REMPipelineManagerConstants.zpoint)
    elif Which(REMPipelineManagerConstants.query) == None:
        parser.error("%s not found." % REMPipelineManagerConstants.query)
#
if options.photometry or options.zeropoint:
    if Which(REMPipelineManagerConstants.gettab) == None:
        parser.error("%s not found." % REMPipelineManagerConstants.gettab)
#
if options.astro:
    if  Which(REMPipelineManagerConstants.astro) == None:
        parser.error("%s not found." % REMPipelineManagerConstants.astro)
    elif Which(REMPipelineManagerConstants.astrochk) == None:
        parser.error("%s not found." % REMPipelineManagerConstants.astrochk)
        


codename = os.path.basename(sys.argv[0])

# Define logging system

if not os.path.isdir(REMPipelineManagerConstants.LogDir):
    try:
        os.mkdir(REMPipelineManagerConstants.LogDir)
    except:
        parser.error("Cannot create log directory %s." % REMPipelineManagerConstants.LogDir)

# create logger
logger = logging.getLogger(codename)
logger.setLevel(logging.DEBUG)

# create console handler
chs = logging.StreamHandler()
chs.setLevel(logging.DEBUG)

# create file handler
chf = logging.handlers.TimedRotatingFileHandler(REMPipelineManagerConstants.LogPath,'D',1,backupCount=30)
chf.setLevel(logging.INFO)

# create a mail handler
chm = logging.handlers.SMTPHandler(REMPipelineManagerConstants.REMSMTP,REMPipelineManagerConstants.FromAdr,
        REMPipelineManagerConstants.ToAdr,REMPipelineManagerConstants.PipeMsg)
chm.setLevel(logging.CRITICAL)

# create formatter
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")

# add formatter
chs.setFormatter(formatter)
chf.setFormatter(formatter)
chm.setFormatter(formatter)

# add handler
if options.verbose:
    logger.addHandler(chs)
logger.addHandler(chf)
logger.addHandler(chm)



if options.database:
    # dbase dir
    if not os.path.isdir(REMPipelineManagerConstants.DBaseLocDir):
        try:
            os.mkdir(REMPipelineManagerConstants.DBaseLocDir)
        except:
            parser.error("Cannot create dbase directory %s." % REMPipelineManagerConstants.DBaseLocDir)
    # create, if it does not exist already a local and a remote dbase
    ldbcr = CreateLocDB(REMPipelineManagerConstants.DBaseLocPath,REMPipelineManagerConstants.DBLocTabName,REMPipelineManagerConstants.DBLocParDict)
    dbcr = CreateRemDB(REMPipelineManagerConstants.DBaseRemAddr,REMPipelineManagerConstants.DBaseRemDB,REMPipelineManagerConstants.DBaseRemUser,
                REMPipelineManagerConstants.DBaseRemPwd,REMPipelineManagerConstants.DBaseRemTabName,REMPipelineManagerConstants.DBaseRemParDict)
    if ldbcr and dbcr:
        logger.info("Database table %s ready." % REMPipelineManagerConstants.DBLocTabName)
    elif not ldbcr:
        logger.error("Local database table %s cannot be created or database cannot be accessed." % REMPipelineManagerConstants.DBLocTabName)
    elif not dbcr:
        logger.error("Remote database table %s cannot be created or database cannot be accessed." % REMPipelineManagerConstants.DBaseRemTabName)
if options.database and options.host:
    # Host data analysis
    ldbcrh = CreateLocDB(REMPipelineManagerConstants.DBaseLocPath,REMPipelineManagerConstants.DBHostTabName,REMPipelineManagerConstants.DBLocParDict)
    dbcrh = CreateRemDB(REMPipelineManagerConstants.DBaseRemAddr,REMPipelineManagerConstants.DBaseRemDB,REMPipelineManagerConstants.DBaseRemUser,
                REMPipelineManagerConstants.DBaseRemPwd,REMPipelineManagerConstants.DBaseHostTabName,REMPipelineManagerConstants.DBaseHostParDict)
    if ldbcrh and dbcrh:
        logger.info("Database table %s ready." % REMPipelineManagerConstants.DBHostTabName)
    elif not ldbcr:
        logger.error("Local database table %s cannot be created or database cannot be accessed." % REMPipelineManagerConstants.DBHostTabName)
    elif not dbcr:
        logger.error("Remote database table %s cannot be created or database cannot be accessed." % REMPipelineManagerConstants.DBaseHostTabName)
    



# Pipeline targets
if not os.path.isdir(REMPipelineManagerConstants.TrgDir):
    try:
        os.mkdir(REMPipelineManagerConstants.TrgDir)
    except:
        parser.error("Cannot create target directory %s." % REMPipelineManagerConstants.TrgDir)




# Starting mjd entry
if options.startingid:
    rossid = options.startingid[0]
    remirid = options.startingid[1]
else:
    dt = LoadProcessPointFName(REMPipelineManagerConstants.ProcessPointFName)
    if dt == None:
        logger.warn("No process point file or badly formatted.")
        rossid = None
        remirid = None
    else:
        # There is a reference file
        logger.info("Reading process point file %s" % REMPipelineManagerConstants.ProcessPointFName)
        rossid = dt[0]
        remirid = dt[1]
    
# Starting frame id
if rossid != None and remirid != None:
    logger.info("Selected starting Id is %d for ROSS and %d for REMIR." % (rossid,remirid))
else:
    logger.info("Starting Ids will be the most recent in table.")



    
oldrossid = rossid
oldremirid = remirid
#
while True:
    # Read catalog table
    logger.info("Logging to %s to get frames to be analyzed" % REMPipelineManagerConstants.REMDBaseAddr)
    frdata = GetREMFrameInfo(REMPipelineManagerConstants.REMDBaseAddr,REMPipelineManagerConstants.REMDBaseDB,
                REMPipelineManagerConstants.REMDBaseUser,REMPipelineManagerConstants.REMDBasePwd,
                REMPipelineManagerConstants.REMDBaseTabName,rossid,remirid)
    if frdata == None:
        logger.critical("Database repository at %s cannot be accessed." % REMPipelineManagerConstants.REMDBaseAddr)
    elif frdata == []:
        logger.info("No new data to be analyzed.")
    else:
        for fr in frdata:
            frmjd = fr[REMPipelineManagerConstants.REMDBaseMJDObs]
            MIDJFRAME = frmjd
            ELTIME = fr[REMPipelineManagerConstants.REMDBaseExp]
            lrsfname = fr[REMPipelineManagerConstants.REMDBaseFName]
            SERIAL = fr[REMPipelineManagerConstants.REMDBaseId]
            #Compute observation date
            if fr[REMPipelineManagerConstants.REMDBaseMJDObs] > 0:
                cmd = REMPipelineManagerConstants.calend + ' -m %.6f ' % (fr[REMPipelineManagerConstants.REMDBaseMJDObs])
                logger.debug("Date computed with command %s" % cmd)
                calendres = Pipe(cmd)
                JulianDate = float(calendres.decode().split()[0])
                YYYYMMDDOBS = calendres.decode().split()[2]
                HHMMSS = calendres.decode().split()[3]
            else:
                YYYYMMDDOBS = '2000/01/01'
                HHMMSS = '00:00:00'
            logger.info("Observations of frame %s carried out on %s %s" % (lrsfname, YYYYMMDDOBS, HHMMSS))
            ymd = YYYYMMDDOBS.replace('/','')
            entryfil = "%s/%s" % (ymd,lrsfname)
            FILENAME = entryfil
            # get image header
            logger.info("Analyzing %s frame (mjd: %.5f)." % (entryfil,frmjd))
            logger.info("Parsing table entries for %s." % entryfil)
            msg = ''
            for i in REMPipelineManagerConstants.REMDBTableList:
                msg = msg + os.linesep + str(i) + ':  '+str(fr[i])
            logger.debug("Chosen table entries for the %s file: %s" % (entryfil, msg))
            # RA,DEC
            RA = fr[REMPipelineManagerConstants.REMDBaseRA]
            DEC = fr[REMPipelineManagerConstants.REMDBaseDEC]
            logger.info("Frame %s coordinates are RA: %.6f and DEC: %.6f." % (entryfil,RA,DEC))     
            # Filter
            FILTER = fr[REMPipelineManagerConstants.REMDBaseFilter]
            logger.info("Frame %s obtained with filter %s." % (entryfil,FILTER))
            # PI
            PI = fr[REMPipelineManagerConstants.REMDBasePI]
            logger.info("Frame %s PI: %s." % (entryfil,PI))
            #
            wcam = WhichREMImagingFrame(fr)
            # OBJECT and OBSCODE
            OBJECT = fr[REMPipelineManagerConstants.REMDBaseObj]            
            if wcam == REMPipelineManagerConstants.ROSS or wcam == REMPipelineManagerConstants.ROS2:
                # ROSS obscodes are one less than REMIR obscodes
                OBSCODE = fr[REMPipelineManagerConstants.REMDBaseObsCode]+1
                rossid = fr[REMPipelineManagerConstants.REMDBaseId]
            elif wcam == REMPipelineManagerConstants.REMIR:
                OBSCODE = fr[REMPipelineManagerConstants.REMDBaseObsCode]
                remirid = fr[REMPipelineManagerConstants.REMDBaseId]
            #
            if wcam != None:
                # REM Imaging frame frame
                logger.info("Frame %s target is %s with code %d." % (entryfil,OBJECT,OBSCODE))
                if wcam == REMPipelineManagerConstants.ROSS:
                    if fr[REMPipelineManagerConstants.REMDBaseFilter] in REMPipelineManagerConstants.ROSSFilterSet: 
                        ROSSFlag = True
                        REMIRFlag = False
                        ROS2Flag = False
                        CAMERA = REMPipelineManagerConstants.ROSS
                        logger.info("%s is a REM-ROSS imaging frame." % entryfil)
                        logger.info("Serial number is %d." % rossid)
                    else:
                        ROSSFlag = False
                        ROS2Flag = False
                        REMIRFlag = False
                        logger.info("%s is a REM-ROSS frame not to be processed." % entryfil)    
                        logger.info("Serial number is %d." % remirid)      
                elif wcam == REMPipelineManagerConstants.ROS2:
                    if fr[REMPipelineManagerConstants.REMDBaseFilter] in REMPipelineManagerConstants.ROSSFilterSet: 
                        ROSSFlag = False
                        REMIRFlag = False
                        ROS2Flag = True
                        CAMERA = REMPipelineManagerConstants.ROS2
                        logger.info("%s is a REM-ROS2 imaging frame." % entryfil)
                        logger.info("Serial number is %d." % rossid)
                    else:
                        ROSSFlag = False
                        ROS2Flag = False
                        REMIRFlag = False
                        logger.info("%s is a REM-ROS2 frame not to be processed." % entryfil)    
                        logger.info("Serial number is %d." % remirid)      
                elif wcam == REMPipelineManagerConstants.REMIR:
                    if fr[REMPipelineManagerConstants.REMDBaseFilter] in REMPipelineManagerConstants.REMIRFilterSet:
                        ROSSFlag = False
                        ROS2Flag = False
                        REMIRFlag = True
                        CAMERA = REMPipelineManagerConstants.REMIR
                        logger.info("%s is a REM-REMIR imaging frame." % entryfil)
                        logger.info("Serial number is %d." % remirid)
                    else:
                        ROSSFlag = False
                        ROS2Flag = False
                        REMIRFlag = False
                        logger.info("%s is a REM-REMIR sub-frame not to be processed." % entryfil)    
                        logger.info("Serial number is %d." % remirid)                    
                if ROSSFlag or ROS2Flag or REMIRFlag:
                    # woraround for Test vs Focus
                    if OBSCODE == REMPipelineManagerConstants.TST:
                        OBSCODE = REMPipelineManagerConstants.FCS
                    # Good imaging frame
                    # Retrieve only those we are interested in
                    if OBSCODE in REMPipelineManagerConstants.OkTargetSet:
                        logger.info("%s is a REM imaging frame to be processed." % entryfil)
                        logger.info("Retrieving REM frame: %s" % entryfil)
                        #
                        attmpt = 0 
                        while GetREMFrame (entryfil,wcam,lrsfname) == None and attmpt < 3:   
                            # retry next time
                            attmpt = attmpt + 1
                            time.sleep(REMPipelineManagerConstants.DwnDelay)
                        #
                        if attmpt == 3:
                            logger.critical("Cannot import %s REM file" % entryfil)
                            break
                        else:
                            logger.info("Local file %s created." % (lrsfname+REMPipelineManagerConstants.locext))
                        # astrometry
                        if options.astro and OBSCODE != REMPipelineManagerConstants.FCS:
                            cmd = REMPipelineManagerConstants.astrochk + ' -i %s' % (lrsfname+REMPipelineManagerConstants.locext)
                            logger.debug("Astrometry checked with command: %s" % cmd)
                            astchkres = Pipe(cmd)
                            try:
                                astrchkflg = int(astchkres.decode().split()[0])
                            except IndexError:
                                astrchkflg = 0
                                logger.error("Unexpected astrometry response to query: %s" % cmd)
                            if astrchkflg:
                                logger.info("Frame %s is correctly astrometrized." % entryfil)
                                astflg = 1
                            else:
                                logger.info("Frame %s is without astrometric solution." % entryfil)
                                res = AstroDeepness (MIDJFRAME,CAMERA)
                                cmd = REMPipelineManagerConstants.astro + ' -i %s -o %s -n %d %d -b %.1f -s %d -a %s' % (lrsfname+REMPipelineManagerConstants.locext, lrsfname+REMPipelineManagerConstants.locext, res['nsrc'], res['ncat'], res['box'], res['steps'], res['ang'])
                                logger.debug("Astrometry computed with command: %s" % cmd)
                                astres = Pipe(cmd)
                                try:
                                    astflg = int(astres.decode().split()[0])
                                except IndexError:
                                    astflg = 0
                                    logger.error("Unexpected astrometry response to query: %s" % cmd)    
                                if astflg:
                                    logger.info("Astrometry for frame %s positively computed." % entryfil)
                                else:
                                    logger.info("Astrometry for frame %s cannot be computed." % entryfil)
                            #
                            # copy frame with astrometry
                            if options.copyastro and astflg:
                                basedir = os.path.dirname(entryfil)
                                if not os.path.exists(os.path.join(options.copyastro,basedir)):
                                    try:
                                        os.makedirs(os.path.join(options.copyastro,basedir))
                                    except IOError:
                                        logger.error("Impossible to create remote directory %s." % os.path.join(options.copyastro,basedir))
                                try:
                                    f_in = open(lrsfname+REMPipelineManagerConstants.locext, 'rb')
                                    f_out = gzip.open(os.path.join(options.copyastro,entryfil+REMPipelineManagerConstants.locext+'.gz'), 'wb')
                                    f_out.writelines(f_in)
                                    f_out.close()
                                    f_in.close()
                                    logger.info("Local astrometrized file copied to %s." % options.copyastro)
                                except IOError:
                                    logger.error("Impossible to copy local astrometrized file to %s." % options.copyastro)
                        #
                        # Header values
                        # DEROT, DWANGLE
                        DEROTANG = GetHeaderValue(lrsfname+REMPipelineManagerConstants.locext,REMPipelineManagerConstants.DEROTANG)[0]
                        logger.info("Derotator offset: %.1f." % DEROTANG)
                        if REMIRFlag:
                            DWANGLE = GetHeaderValue(lrsfname+REMPipelineManagerConstants.locext,REMPipelineManagerConstants.DWANGLE)[0]
                            logger.info("Dither wedge position: %.1f." % DWANGLE)
                        else:
                            DWANGLE = -99.
                        #
                        if wcam == REMPipelineManagerConstants.REMIR:
                            AZ,ALT,AZc,ALTc,POINT,PANGLE = ComputeREMAltAz(lrsfname+REMPipelineManagerConstants.locext,{'x': REMPipelineManagerConstants.REMIRDC['x'],'y': REMPipelineManagerConstants.REMIRDC['y']})
                        else:
                            if 'BL' in lrsfname:
                                AZ,ALT,AZc,ALTc,POINT,PANGLE = ComputeREMAltAz(lrsfname+REMPipelineManagerConstants.locext,{'x': REMPipelineManagerConstants.ROS2BLDC['x'],'y': REMPipelineManagerConstants.ROS2BLDC['y']})
                            elif 'BR' in lrsfname:
                                AZ,ALT,AZc,ALTc,POINT,PANGLE = ComputeREMAltAz(lrsfname+REMPipelineManagerConstants.locext,{'x': REMPipelineManagerConstants.ROS2BRDC['x'],'y': REMPipelineManagerConstants.ROS2BRDC['y']})
                            elif 'UL' in lrsfname:
                                AZ,ALT,AZc,ALTc,POINT,PANGLE = ComputeREMAltAz(lrsfname+REMPipelineManagerConstants.locext,{'x': REMPipelineManagerConstants.ROS2ULDC['x'],'y': REMPipelineManagerConstants.ROS2ULDC['y']})
                            elif 'UR' in lrsfname:
                                AZ,ALT,AZc,ALTc,POINT,PANGLE = ComputeREMAltAz(lrsfname+REMPipelineManagerConstants.locext,{'x': REMPipelineManagerConstants.ROS2URDC['x'],'y': REMPipelineManagerConstants.ROS2URDC['y']})
                            else:
                                AZ,ALT,AZc,ALTc,POINT,PANGLE = ComputeREMAltAz(lrsfname+REMPipelineManagerConstants.locext)
                        logger.debug("REM log files data: %s %s %s %s %s %s" % (AZ,ALT,AZc,ALTc,POINT,PANGLE))
                        #try:
                        #    AZ = float(res1)
                        #    ALT = float(res2)
                        if AZ != None:
                                logger.info("Target AZ and ALT of frame %s are %.6f %.6f" % (entryfil,AZ,ALT))
                                logger.info("Pointed AZ and ALT of frame %s are %.6f %.6f" % (entryfil,AZc,ALTc))
                                logger.info("Telescope pointing mode: %s" % (REMPipelineManagerConstants.PointDict[POINT]))
                                logger.info("Parallactic angle for frame %s: %.1f" % (entryfil,PANGLE))
                        #except ValueError:
                        #    AZ = 0.0
                        #    ALT = 0.0
                        else:
                            AZ = 0.0
                            ALT = 0.0
                            AZc = 0.0
                            ALTc = 0.0
                            POINT = 0.0
                            PANGLE = 0.0
                            logger.warn("In file %s header badly formatted." % entryfil)
                        # Internal temperature and focus
                        ITEMP = GetREMTemp(JulianDate)
                        logger.info("Internal temperature for frame %s: %.2f" % (entryfil,ITEMP))
                        FOCUS = GetREMFocusData(JulianDate)
                        logger.info("Focus position for frame %s: %.2f" % (entryfil,FOCUS))
                        #
                        # Astrometry & photometry
                        PROCSTAT = -99
                        PHOTSTAB = -99
                        if ROSSFlag or ROS2Flag:
                            res = GetHeaderValue(lrsfname+REMPipelineManagerConstants.locext,REMPipelineManagerConstants.PROCSTAT)[0]
                            try:
                                PROCSTAT = int(res)
                                logger.info("PROCSTAT for frame %s: %d" % (entryfil,PROCSTAT))
                            except ValueError:
                                logger.warn("In file %s PROCSTAT badly formatted." % entryfil)
                            #
                            if ROS2Flag:
                                # Modify MJD to avoid to have the same entries.
                                if FILTER == REMPipelineManagerConstants.g:
                                    MIDJFRAME = MIDJFRAME + 1e-7
                                elif FILTER == REMPipelineManagerConstants.r:
                                    MIDJFRAME = MIDJFRAME + 2e-7
                                elif FILTER == REMPipelineManagerConstants.i:
                                    MIDJFRAME = MIDJFRAME + 3e-7
                                elif FILTER == REMPipelineManagerConstants.z:
                                    MIDJFRAME = MIDJFRAME + 4e-7
                                else:
                                    logger.error("Filter %s for frame %s not recognized" % (FILTER, entryfil))
                        elif REMIRFlag:
                            res1 = GetHeaderValue(lrsfname+REMPipelineManagerConstants.locext,REMPipelineManagerConstants.PHOTSTAB)[0]
                            try:
                                PHOTSTAB = float(res1)
                                logger.info("PHOTSTAB for frame %s: %d" % (entryfil,PHOTSTAB))
                            except ValueError:
                                logger.warn("In file %s PHOTSTAB badly formatted." % entryfil)
                        # MidMJD, Eltime
                        if REMIRFlag:
                            res1 = GetHeaderValue(lrsfname+REMPipelineManagerConstants.locext,REMPipelineManagerConstants.TMJDOBS)[0]
                            res2 = GetHeaderValue(lrsfname+REMPipelineManagerConstants.locext,REMPipelineManagerConstants.TEXP)[0]                
                            try:
                                MIDJFRAME = float(res1)
                                ELTIME = float(res2)
                                logger.info("Mid modified JD and elapsed time of frame %s are %.6f %.6f" % (entryfil,MIDJFRAME,ELTIME))
                            except ValueError:
                                logger.warn("In file %s mid JD and elapsed time badly formatted." % entryfil)  
                            except TypeError:
                                logger.warn("In file %s mid JD and elapsed time badly formatted." % entryfil)
                        #
                        # extinction coeffienct and tables
                        extcoeff = REMPipelineManagerConstants.LaSillaExtCoeff[FILTER]
                        #
                        if OBSCODE == REMPipelineManagerConstants.AGN:
                            filtcol = REMPipelineManagerConstants.REMFiltColDictAGN[FILTER]
                            tabcat = REMPipelineManagerConstants.REMFiltCatDictAGN[FILTER]                        
                        else:
                            filtcol = REMPipelineManagerConstants.REMFiltColDict[FILTER]
                            tabcat = REMPipelineManagerConstants.REMFiltCatDict[FILTER]
                        #
            
                        # photometry
                        NPHOTOSTARS = 0
                        FWHMFRAME = -1
                        #
                        if options.photometry:
                        #
                            if CAMERA == REMPipelineManagerConstants.ROS2:
                                cmd = REMPipelineManagerConstants.param + ' -p REM-ROS2'
                            elif CAMERA == REMPipelineManagerConstants.ROSS:
                                cmd = REMPipelineManagerConstants.param + ' -p REM-ROSS'
                            elif CAMERA == REMPipelineManagerConstants.REMIR:
                                cmd = REMPipelineManagerConstants.param + ' -p REM-REMIR'                        
                            logger.debug("SExtractor parameter file generated with command: %s" % cmd)
                            Pipe(cmd)
                            if CAMERA == REMPipelineManagerConstants.ROS2:
                                cmd = REMPipelineManagerConstants.photo + ' -i %s -g %.2f -s %.2f ' % (lrsfname+REMPipelineManagerConstants.locext,
                                    REMPipelineManagerConstants.ROS2GAIN, REMPipelineManagerConstants.ROSSSATURATION)
                            elif CAMERA == REMPipelineManagerConstants.ROSS:
                                cmd = REMPipelineManagerConstants.photo + ' -i %s -g %.2f -s %.2f ' % (lrsfname+REMPipelineManagerConstants.locext,
                                    REMPipelineManagerConstants.ROSSGAIN, REMPipelineManagerConstants.ROSSSATURATION)
                            elif CAMERA == REMPipelineManagerConstants.REMIR:
                                cmd = REMPipelineManagerConstants.photo + ' -i %s -g %.2f -s %.2f ' % (lrsfname+REMPipelineManagerConstants.locext,
                                    REMPipelineManagerConstants.REMIRGAIN, REMPipelineManagerConstants.REMIRSATURATION)
                            logger.debug("Photometry derived with command: %s" % cmd)
                            photores = Pipe(cmd)
                            NPHOTOSTARS = int(photores.decode().split()[0])
                            photooutfile = photores.decode().split()[1]
                            logger.info("%d stars in frame %s photometrized." % (NPHOTOSTARS,entryfil))
                            if NPHOTOSTARS > 0:
                                cmd = REMPipelineManagerConstants.sigmak + ' -i %s -d 12' % (photooutfile)
                                logger.debug("FWHM estimated with command: %s" % cmd)
                                fwhmres = Pipe(cmd)
                                FWHMFRAME = float(fwhmres.decode().split()[0])
                                logger.info("FWHM for frame %s: %.2f pixel." % (entryfil, FWHMFRAME))

                        # zeropoint
                        ZPMAG = -99
                        ZPMAGERR = -99
                        ZPNSTARS = 0
                        CATNSTARS = 0
                        #
                        if options.zeropoint and NPHOTOSTARS > 0:
                            #
                            cmd = REMPipelineManagerConstants.query + ' -f %s -r %.1f -o %s -C %s -m 500' % ((lrsfname+REMPipelineManagerConstants.locext),
                                REMPipelineManagerConstants.REMSIZE, REMPipelineManagerConstants.CatOutFile, tabcat)
                            logger.debug("Standard star catalog query executed with command: %s" % cmd)
                            queryres = Pipe(cmd)
                            try:
                                CATNSTARS = int(queryres.decode().split()[0])
                            except ValueError:
                                CATNSTARS = 0
                                logger.error("Unexpected catalogue response to query: %s" % cmd)
                            except IndexError:
                                CATNSTARS = 0
                                logger.error("Unexpected catalogue response to query: %s" % cmd)
                            #
                            logger.info("Number of standard stars found: %d." % CATNSTARS)                    
                            if CATNSTARS > 0:
                                if OBSCODE == REMPipelineManagerConstants.STD:
                                    # standard star, magnitude to infinity
                                    cmd = REMPipelineManagerConstants.zpoint + ' -i %s -c %s -I 1 4 5 10 11 14 -C 1 2 3 %d %d -a %s -t %.1f' % (photooutfile,
                                        REMPipelineManagerConstants.CatOutFile, filtcol[0], filtcol[1], extcoeff, 5)
                                else:
                                    # all other targets, aperture magnitude (now, again magnitude to infinity)
                                    cmd = REMPipelineManagerConstants.zpoint + ' -i %s -c %s -I 1 4 5 10 11 14 -C 1 2 3 %d %d -a %s -t %.1f' % (photooutfile,
                                        REMPipelineManagerConstants.CatOutFile, filtcol[0], filtcol[1], extcoeff, 5)
                                logger.debug("Zero-point computation executed with command: %s" % cmd)
                                zerores = Pipe(cmd)
                                ZPMAG = float(zerores.decode().split()[0])
                                ZPMAGERR = float(zerores.decode().split()[1])
                                ZPNSTARS = int(zerores.decode().split()[2])
                                if ZPNSTARS == 0:
                                    ZPMAG = -99
                                    ZPMAGERR = -99
                                    logger.warning("Not enough standard stars / frame stars matches.")
                                else:
                                    logger.info("Frame %s zero-point in filter %s is %.3f +/- %.3f derived with %d stars." % (entryfil, FILTER, ZPMAG, ZPMAGERR, ZPNSTARS))
                    
                        #
                        OBJMAG = -99
                        OBJMAGERR = -99
                        XPOS = -99
                        YPOS = -99
                        #
                        # AGN
                        if OBSCODE == REMPipelineManagerConstants.AGN:
                            if NPHOTOSTARS > 0 and ZPMAG > -90:
                                # Only for good data.
                                logger.info("Frame %s is an AGN frame." % entryfil)
                                trglist = LoadREMTargets(REMPipelineManagerConstants.TrgAGNPath)
                                if trglist != None:
                                    # find selected objects in photometry file
                                    for tg in trglist:
                                        agname = tg[0]
                                        agnra = tg[1]
                                        agndec = tg[2]
                                        logger.info("Target %s of coordinates RA=%.6f DEC=%.6f in AGN list." % (agname, agnra, agndec))
                                        #
                                        cmd = REMPipelineManagerConstants.gettab + ' -i %s -c 4 5 -C %.6f %.6f -t %.5f -a' % (photooutfile,
                                            agnra, agndec, 3.0/3600.0)
                                        logger.debug("Selected AGN target searched with command: %s" % cmd)
                                        selres = Pipe(cmd)
                                        selresl = selres.decode().split(os.linesep)
                                        try:
                                            nbs = int(selresl[0])
                                        except ValueError:
                                            nbs = 0
                                            logger.critical("Selected table entry error in photometry file %s." % photooutfile)
                                        if nbs > 0:
                                            logger.info("%d object(s) selected." % nbs)
                                            try:
                                                amag = float(selresl[1].split()[9])
                                                eamag = float(selresl[1].split()[10])
                                                XPOS = float(selresl[1].split()[1])
                                                YPOS = float(selresl[1].split()[2])
                                            except ValueError:
                                                amag = -99
                                                eamag = -99
                                                XPOS = -99
                                                YPOS = -99
                                                logger.critical("Cannot read object data from photometry file %s" % photooutfile)
                                            OBJMAG = amag -25.0 + ZPMAG
                                            OBJMAGERR = math.sqrt(eamag**2 + ZPMAGERR**2)
                                            OBJECT = agname
                                            logger.info("Object %s has magnitude %.3f +/- %.3f in filter %s" % (OBJECT, OBJMAG, OBJMAGERR, FILTER)) 
                                            logger.info("Object %s is at position X: %.2f and Y: %.2f" % (OBJECT, XPOS, YPOS)) 
                                            break
                                        #
                                else:
                                    logger.warning("No AGN target file %s or file not readable." % REMPipelineManagerConstants.TrgAGNPath)
                        
                        # GRB
                        if OBSCODE in REMPipelineManagerConstants.GRBTargetSet:
                            if NPHOTOSTARS > 0:
                                # Only for good data.
                                logger.info("Frame %s is a GRB frame." % entryfil)
                                grblist = GetGRBInfo(OBJECT)
                                if grblist == None:
                                    logger.error("GRB data not accessible.")
                                else:
                                    if 0 <= grblist[3] < 10:
                                        logger.info("GRB accurate information available.")
                                        logger.info("Event: %s, RA: %.6f, DEC: %.6f, Err.: %.2f, Burst time: %.6f." % (grblist[0],grblist[1],grblist[2],grblist[3],grblist[4]))
                                        #
                                        if grblist[3] < 3:
                                            grberr = 3.0
                                        else:
                                            grberr = grblist[3]
                                        #
                                        cmd = REMPipelineManagerConstants.gettab + ' -i %s -c 4 5 -C %.6f %.6f -t %.5f -a' % (photooutfile,
                                            grblist[1], grblist[2], math.ceil(grberr)/3600.0)
                                        logger.debug("GRB target searched with command: %s" % cmd)
                                        selres = Pipe(cmd)
                                        selresl = selres.decode().split(os.linesep)
                                        try:
                                            nbs = int(selresl[0])
                                        except ValueError:
                                            nbs = 0
                                            logger.critical("Selected table entry error in photometry file %s." % photooutfile)
                                        logger.info("%d object(s) selected." % nbs)
                                        if nbs > 0:
                                            try:
                                                amag = float(selresl[1].split()[9])
                                                eamag = float(selresl[1].split()[10])
                                                XPOS = float(selresl[1].split()[1])
                                                YPOS = float(selresl[1].split()[2])
                                            except ValueError:
                                                amag = -99
                                                eamag = -99
                                                XPOS = -99
                                                YPOS = -99
                                                logger.critical("Cannot read object data from photometry file %s" % photooutfile)
                                            if CAMERA == REMPipelineManagerConstants.ROSS or CAMERA == REMPipelineManagerConstants.ROS2: 
                                                ZPMAG = REMPipelineManagerConstants.RossZP[FILTER][0]
                                                ZPMAGERR = REMPipelineManagerConstants.RossZP[FILTER][1]
                                            elif CAMERA == REMPipelineManagerConstants.REMIR and FILTER == REMPipelineManagerConstants.Z:
                                                ZPMAG = REMPipelineManagerConstants.RemirZP[FILTER][0]
                                                ZPMAGERR = REMPipelineManagerConstants.RemirZP[FILTER][1]
                                            #
                                            OBJMAG = amag -25.0 + ZPMAG
                                            OBJMAGERR = math.sqrt(eamag**2 + ZPMAGERR**2)
                                            OBJECT = grblist[0]
                                            # Compute observation julian date
                                            cmd = REMPipelineManagerConstants.calend + ' -m %.6f ' % (MIDJFRAME)
                                            logger.debug("Julian date computed with command %s" % cmd)
                                            calendres = Pipe(cmd)
                                            bjd = float(calendres.split()[0])
                                            #
                                            if nbs > 1:
                                                logger.info("There are %d objects in error circle. Selecting the closest." % nbs) 
                                            logger.info("Object %s has magnitude %.3f +/- %.3f at %.0f s from burst in filter %s in frame %s" % (OBJECT, OBJMAG, OBJMAGERR, (bjd-grblist[4])*86400.0, FILTER, entryfil))
                                            logger.info("Object %s is at position X: %.2f and Y: %.2f" % (OBJECT, XPOS, YPOS)) 
                                            #
                                            # Mail to friends
                                            msg1 = "Object %s has magnitude %.3f +/- %.3f at %.0f s from burst in filter %s in frame %s." % (OBJECT, OBJMAG, OBJMAGERR, (bjd-grblist[4])*86400.0, FILTER, entryfil)
                                            msg1b = "Object %s is at position X: %.2f and Y: %.2f" % (OBJECT, XPOS, YPOS)
                                            if nbs > 1:
                                                msg2 = "%d objects in error cicle. Selected the closest." % nbs
                                            else:
                                                msg2 = ""
                                            sbj = "%s REM automatic analysis result" % OBJECT
                                            SendEMail(msg1+os.linesep+msg1b+os.linesep+msg2,REMPipelineManagerConstants.ToGRBAdr,REMPipelineManagerConstants.REMSMTP,sbj,REMPipelineManagerConstants.FromAdr)
                                            #
                                    else:
                                        logger.info("No available accurate information for %s" % OBJECT)


                        if options.host:
                            # Vitali
                            if OBSCODE == REMPipelineManagerConstants.AGN and PI == 'Fabrizio_Vitali':
                                if NPHOTOSTARS > 0 and ZPMAG > -90 and CAMERA == REMPipelineManagerConstants.REMIR:
                                    # Only for NIR and good data.
                                    logger.info("Frame %s is a NIR AGN frame with PI Fabrizio Vitali." % entryfil)
                                    trglist = LoadREMTargets(REMPipelineManagerConstants.TrgVitaliPath)
                                    if trglist != None:
                                        # find selected objects in photometry file
                                        for tg in trglist:
                                            agname = tg[0]
                                            agnra = tg[1]
                                            agndec = tg[2]
                                            logger.info("Target %s of coordinates RA=%.6f DEC=%.6f in Vitali's list." % (agname, agnra, agndec))
                                            #
                                            cmd = REMPipelineManagerConstants.gettab + ' -i %s -c 4 5 -C %.6f %.6f -t %.5f -a' % (photooutfile,
                                                agnra, agndec, 3.0/3600.0)
                                            logger.debug("Selected Vitali's target searched with command: %s" % cmd)
                                            selres = Pipe(cmd)
                                            selresl = selres.split(os.linesep)
                                            try:
                                                nbs = int(selresl[0])
                                            except ValueError:
                                                nbs = 0
                                                logger.critical("Selected table entry error in photometry file %s." % photooutfile)
                                            if nbs > 0:
                                                logger.info("%d object(s) selected." % nbs)
                                                try:
                                                    amag = float(selresl[1].split()[5])
                                                    eamag = float(selresl[1].split()[6])
                                                    XPOS = float(selresl[1].split()[1])
                                                    YPOS = float(selresl[1].split()[2])
                                                except ValueError:
                                                    amag = -99
                                                    eamag = -99
                                                    XPOS = -99
                                                    YPOS = -99
                                                    logger.critical("Cannot read object data from photometry file %s" % photooutfile)
                                                OBJMAGH = amag -25.0 + ZPMAG
                                                OBJMAGERRH = math.sqrt(eamag**2 + ZPMAGERR**2)
                                                OBJECTH = agname
                                                logger.info("Object %s has magnitude %.3f +/- %.3f" % (OBJECTH, OBJMAGH, OBJMAGERRH))
                                                logger.info("Object %s is at position X: %.2f and Y: %.2f" % (OBJECTH, XPOS, YPOS)) 
                                                if options.database:
                                                    dbargh = (FILENAME,OBJECTH,CAMERA,FILTER,NPHOTOSTARS,FWHMFRAME,ZPMAG,ZPMAGERR,ZPNSTARS,
                                                        OBJMAGH,OBJMAGERRH,MIDJFRAME,ELTIME,PI,XPOS,YPOS,PROCSTAT,PHOTSTAB)
                                                    ldbsavh = SaveResult2LocDB(REMPipelineManagerConstants.DBaseLocPath,REMPipelineManagerConstants.DBHostTabName,dbargh)
                                                    dbsavh = SaveResult2RemDB(REMPipelineManagerConstants.DBaseRemAddr,REMPipelineManagerConstants.DBaseRemDB,
                                                        REMPipelineManagerConstants.DBaseRemUser,REMPipelineManagerConstants.DBaseRemPwd,REMPipelineManagerConstants.DBaseHostTabName,dbargh)
                                                    if dbsavh and ldbsavh:
                                                        logger.info("Host database data for frame %s saved." % entryfil)
                                                    elif not dbsavh:
                                                        logger.error("Remote host database data for frame %s could not be saved." % entryfil)
                                                    elif not ldbsavh:
                                                        logger.error("Local host database data for frame %s could not be saved." % entryfil)                                 
                                            #
                                    else:
                                        logger.warning("No AGN target file %s or file not readable." % REMPipelineManagerConstants.TrgVitaliPath)


                        
                        # database
                        if options.database:
                            #
                            dbarg = (FILENAME,OBJECT,REMPipelineManagerConstants.REMTargetDict[OBSCODE],RA,DEC,SERIAL,CAMERA,
                                FILTER,NPHOTOSTARS,FWHMFRAME,ZPMAG,ZPMAGERR,ZPNSTARS,AZ,ALT,OBJMAG,OBJMAGERR,MIDJFRAME,ELTIME,XPOS,YPOS,PROCSTAT,
                                PHOTSTAB,AZc,ALTc,POINT,ITEMP,FOCUS,DEROTANG,DWANGLE,PANGLE)
                            ldbsav = SaveResult2LocDB(REMPipelineManagerConstants.DBaseLocPath,REMPipelineManagerConstants.DBLocTabName,dbarg)
                            dbsav = SaveResult2RemDB(REMPipelineManagerConstants.DBaseRemAddr,REMPipelineManagerConstants.DBaseRemDB,
                                REMPipelineManagerConstants.DBaseRemUser,REMPipelineManagerConstants.DBaseRemPwd,REMPipelineManagerConstants.DBaseRemTabName,dbarg)
                            if dbsav and ldbsav:
                                logger.info("Database data for frame %s saved." % entryfil)
                            elif not dbsav:
                                logger.error("Remote database data for frame %s could not be saved." % entryfil)
                            elif not ldbsav:
                                logger.error("Local database data for frame %s could not be saved." % entryfil)                                    
                                    
                                                                
                        # remove processed file
                        if not options.keep:
                            os.remove(lrsfname+REMPipelineManagerConstants.locext)
                            logger.info("Local file %s removed." % lrsfname)
                        if os.path.exists(REMPipelineManagerConstants.CatOutFile):
                            os.remove(REMPipelineManagerConstants.CatOutFile)
                        if os.path.exists(lrsfname+REMPipelineManagerConstants.PhotOutFile):
                            os.remove(lrsfname+REMPipelineManagerConstants.PhotOutFile)
    
    
                            
            else:
                # any other file
                logger.warning("It is not possible to classify frame %s." % entryfil)
    
            #                
            # Upgrade of the processed file name
            sp = SaveProcessPointFName(REMPipelineManagerConstants.ProcessPointFName,rossid,remirid)
            logger.info("Process point file %s updated at %s and %s." % (REMPipelineManagerConstants.ProcessPointFName,str(rossid),str(remirid)))
            if not sp:
                logger.error("Process file %s cannot be saved." % REMPipelineManagerConstants.ProcessPointFName)
                                        
    # delay
    if rossid == oldrossid and remirid == oldremirid:
        time.sleep(REMPipelineManagerConstants.TimeDelay)
    else:
        oldrossid = rossid
        oldremirid = remirid
