Adding data ingestion script and resources

data_ingestion.py

+
+import xml.etree.ElementTree as ET
+import glob
+from datetime import datetime
+from imagedb.models import *
+
+flist = glob.glob('resources/*.xml')
+
+try:
+  inst = Instrument.objects.get(instrumentName="NISP")
+except:
+  inst = Instrument.objects.create(telescopeName='Euclid', instrumentName='NISP')
+   
+
+for f in flist:
+  root = ET.parse(f)
+  exposureTime = float(root.find(".//ExposureTime/Value").text)
+  print(root.find(".//OBT").text) 
+  observationDate = datetime.strptime(root.find(".//OBT").text, 
+                                      "%Y-%m-%dT%H:%M:%S.%f")
+  obs_id = int(root.find(".//FieldId").text)
+  dither = int(root.find(".//DitherObservation").text)
+  ra = float(root.find(".//TargetPointing/RA").text)
+  dec = float(root.find(".//TargetPointing/Dec").text)
+  orientation = float(root.find(".//TargetPointing/Orientation").text)
+  filterWheelPos = root.find(".//Filter").text[4]
+  file_name = root.find(".//DataStorage/DataContainer/FileName").text
+  file_name = file_name.replace("NIR_W-CALIB","LE1_NISP")
+  
+  data = DataContainer(
+    fileFormat = 'fits',
+    formatIdentifier = 'le1.nisprawframe',
+    formatVersion = '1.0',
+    url = "http://ia2-owncloud.oats.inaf.it/fake/7ff2f203/data/" + file_name    
+  )
+  data.save()
+  
+  image = NispRawFrame(
+    exposureTime = exposureTime,
+    imgNumber = 16,
+    naxis1 = 2040,
+    naxis2 = 2040,
+    imageType = {'category':'SCIENCE', 
+                 'firstType':'OBJECT', 
+                 'secondType':'SKY'},
+    observationDateTime = observationDate,
+    observationId = obs_id,
+    ditherNumber = dither,
+    instrument = inst,
+    commandedPointing = {'rightAscension':ra,
+                         'declination':dec,
+                         'orientation':orientation},
+    filterWheelPosition = filterWheelPos,
+    grismWheelPosition = 'OPEN'   
+  )
+  
+  image.frameFile = data
+  image.save()
+  
+  detList = root.findall('.//Detector')
+  detList = sorted(detList, key = lambda x : x.find(".//DetectorId").text)
+  
+  for d in detList:
+    det_id = d.find(".//DetectorId").text
+    gain = float(d.find(".//Gain").text)
+    readoutNoise = float(d.find(".//ReadoutNoise").text)
+
+    detector = image.detectors.create(
+      detectorId = det_id,
+      gain = gain,
+      readoutNoise = readoutNoise
+    )
+    
+    coordinate1 = d.find(".//CTYPE1/CoordinateType").text
+    projection1 = d.find(".//CTYPE1/ProjectionType").text
+    
+    coordinate2 = d.find(".//CTYPE2/CoordinateType").text
+    projection2 = d.find(".//CTYPE2/ProjectionType").text
+    
+    crval1 = float(d.find(".//CRVAL1").text)
+    crval2 = float(d.find(".//CRVAL2").text)
+    crpix1 = float(d.find(".//CRPIX1").text)
+    crpix2 = float(d.find(".//CRPIX2").text)
+    cd1_1 = float(d.find(".//CD1_1").text)
+    cd1_2 = float(d.find(".//CD1_2").text)
+    cd2_1 = float(d.find(".//CD2_1").text)
+    cd2_2 = float(d.find(".//CD2_2").text)
+    
+    Astrometry.objects.create(
+      ctype1 = {'coordinateType': coordinate1, 'projectionType': 'TAN'},
+      ctype2 = {'coordinateType': coordinate2, 'projectionType': 'TAN'},
+      crval1 = crval1,
+      crval2 = crval2,
+      crpix1 = crpix1,
+      crpix2 = crpix2,
+      cd1_1 = cd1_1,
+      cd1_2 = cd1_2,
+      cd2_1 = cd2_1,
+      cd2_2 = cd2_2,
+      detector = detector
+    )
+