ServletCutout.java

import java.util.logging.Logger;
import java.util.logging.Level;

import java.security.Principal;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.ServletOutputStream;

import java.io.OutputStreamWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;

/* for streaming the cutout-file */
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;

import java.util.Arrays;
import java.util.List;
import java.util.LinkedList;
import java.util.Map;
import java.util.HashMap;
import java.util.Properties;

// for Logging/Accounting
import org.json.simple.JSONObject;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimeZone;


import java.nio.file.StandardOpenOption;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

import vo.parameter.*;
import vo.error.*;

public class ServletCutout extends HttpServlet
{
   protected static final Logger      LOGGER     = Logger.getLogger(ServletCutout.class.getName());
   protected static final Settings    settings   = Settings.getInstance();
   protected static final Subsurvey[] subsurveys = Subsurvey.loadSubsurveys(settings.fitsPaths.surveysMetadataAbsPathname());

   protected boolean resolveFromId    = true;//FIXME separate setting authz is separate table settings.dbConn.isDbUriEmpty(); 
   protected boolean useEngineOverCli = settings.amqpConn.isHostnameEmpty();

   final String RESPONSE_ENCODING      = "utf-8";
   final String DEFAULT_RESPONSEFORMAT = "application/fits";

   public void init() throws ServletException
   {
      LOGGER.config("FITS : " + settings.fitsPaths.toString());
      if(subsurveys != null)
         LOGGER.config("Subsurveys loaded : " + String.valueOf(subsurveys.length));
      LOGGER.config("DEFAULT_RESPONSEFORMAT : " + DEFAULT_RESPONSEFORMAT);
      LOGGER.config("Resolver : " + (resolveFromId    ? "IVOID" : "DB"));
      LOGGER.config("Engine   : " + (useEngineOverCli ? "CLI"   : "AMQP"));
      if(!useEngineOverCli)
         LOGGER.config("AMQP : " + settings.amqpConn.toString());
   }


   protected void doSodaDescriptor(PrintWriter writer, String requestUrl)
   {
      String theDescriptor =
         "<VOTABLE xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns=\"http://www.ivoa.net/xml/VOTable/v1.3\" version=\"1.3\">"
         + "<RESOURCE type=\"meta\" utype=\"adhoc:service\" name=\"this\">"
         + "<PARAM name=\"standardID\" datatype=\"char\" arraysize=\"*\" value=\"ivo://ivoa.net/std/SODA#sync-1.0\"/>"
         + "<PARAM name=\"accessURL\" datatype=\"char\" arraysize=\"*\" value=\"" + requestUrl  + "\"/>"
         + "<GROUP name=\"inputParams\">"
         +  "<PARAM name=\"ID\" ucd=\"meta.id;meta.dataset\" datatype=\"char\" arraysize=\"*\" value=\"\"/>"
         +  "<PARAM name=\"POS\" ucd=\"pos.outline;obs\" datatype=\"char\" arraysize=\"*\" value=\"\"/>"
         +  "<PARAM name=\"CIRCLE\" ucd=\"phys.angArea;obs\" unit=\"deg\" datatype=\"double\" arraysize=\"3\" xtype=\"circle\" value=\"\"/>"
         +  "<PARAM name=\"POLYGON\" unit=\"deg\" ucd=\"pos.outline;obs\" datatype=\"double\" arraysize=\"*\" xtype=\"polygon\"  value=\"\"/>"
         +  "<PARAM name=\"BAND\" ucd=\"stat.interval\" unit=\"m\" datatype=\"double\" arraysize=\"2\" xtype=\"interval\" value=\"\"/>"
         +  "<PARAM name=\"TIME\" ucd=\"time.interval;obs.exposure\" unit=\"d\" datatype=\"double\" arraysize=\"2\" xtype=\"interval\" value=\"\"/>"
         +  "<PARAM name=\"POL\" ucd=\"meta.code;phys.polarization\" datatype=\"char\" arraysize=\"*\" value=\"\"/>"
         +  "<PARAM name=\"PIXELS\" ucd=\"instr.pixel;meta.dataset\" datatype=\"char\" arraysize=\"*\" value=\"\"/>"
         +  "<PARAM name=\"RESPONSEFORMAT\" ucd=\"meta.code.mime\" datatype=\"char\" arraysize=\"*\" value=\"application/fits\"/>"

         +  "<PARAM name=\"POSSYS\" ucd=\"pos.frame\" datatype=\"char\" arraysize=\"*\" value=\"\">"
         +   "<DESCRIPTION>Coordinate system for POS values</DESCRIPTION>"
         +   "<VALUES>"
         +     "<OPTION>ICRS</OPTION>"
         +     "<OPTION>GALACTIC</OPTION>"
         +     "<OPTION>GRID</OPTION>"
         +   "</VALUES>"
         +  "</PARAM>"

         +  "<PARAM name=\"BANDSYS\" ucd=\"spect;pos.frame\" datatype=\"char\" arraysize=\"*\" value=\"\">"
         +   "<DESCRIPTION>Coordinate system for BAND vlaues.</DESCRIPTION>"
         +   "<VALUES>"
         +     "<OPTION>WAVE_Barycentric</OPTION>"
         +     "<OPTION>VELO_LSRK</OPTION>"
         +     "<OPTION>GRID</OPTION>"
         +   "</VALUES>"
         +  "</PARAM>"

         + "</GROUP>"
         + "</RESOURCE>"
         + "</VOTABLE>";

      writer.println(theDescriptor);
   }


   protected void doCutoutStream(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
         OutputStream respOutputStream) throws IOException, InterruptedException
   {
      LOGGER.fine("trace " + pos);

      final Resolver resolver = (resolveFromId ?
            new ResolverFromId(subsurveys) :
            new ResolverByObsCore(settings.dbConn, subsurveys));

      final Soda soda = new SodaImpl(settings.fitsPaths);

      resolver.resolve(id);

      soda.doStream(resolver.relPathname(), resolver.hdunum(), pos, band, time, pol, pixels, respOutputStream);
   }


   protected CutResult doCutoutFile(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
         boolean countNullValues)
         throws IOException, InterruptedException
      {
         LOGGER.fine("trace");

         FitsCard[] extraCards = null;

         final Resolver resolver = (resolveFromId ?
               new ResolverFromId(subsurveys)
               : new ResolverByObsCore(settings.dbConn, subsurveys));
         final Vlkb vlkb = (useEngineOverCli ?
               new VlkbCli(settings, subsurveys)
               : new VlkbAmqp(settings, subsurveys));

         resolver.resolve(id);

         String subsurveyId = resolver.obsCollection();
         if(subsurveyId != null)
         {
            extraCards = Subsurvey.subsurveysFindCards(subsurveys, subsurveyId);
         }
         else
         {
            LOGGER.fine("Resolver returns subsurveyId null: no extraCards loaded.");
         }

         return vlkb.doFile(resolver.relPathname(), resolver.hdunum(),
               pos, band, time, pol, pixels, countNullValues, extraCards);
      }



   /* HTTP/J2EE -> SODA */


   /* DALI allows GET and POST for sync services */

   protected void doGet(HttpServletRequest request, HttpServletResponse response)
         throws ServletException, IOException, UnsupportedEncodingException
      {
         final boolean NO_QUERY_STRING = (request.getQueryString() == null);

         if(NO_QUERY_STRING)
         {
            writeSodaDescriptor(request, response);
            LOGGER.fine("normal exit with SODA service descriptor");
            return;
         }
         else
         {
            LOGGER.info(URLDecoder.decode(request.getQueryString(), "UTF-8"));
            execRequest(request, response);
            LOGGER.fine("normal exit");
         }
      }

   protected void doPost(HttpServletRequest request, HttpServletResponse response)
         throws ServletException, IOException, UnsupportedEncodingException
      {
         final boolean NO_QUERY_STRING = (request.getQueryString() == null);

         if(NO_QUERY_STRING)
         {
            writeSodaDescriptor(request, response);
            LOGGER.fine("normal exit with SODA service descriptor");
            return;
         }
         else
         {
            LOGGER.info(URLDecoder.decode(request.getQueryString(), "UTF-8"));
            execRequest(request, response);
            LOGGER.fine("normal exit");
         }
      }



   protected void writeSodaDescriptor(HttpServletRequest request, HttpServletResponse response)
         throws ServletException, IOException, UnsupportedEncodingException
      {
         PrintWriter writer = new PrintWriter(new OutputStreamWriter(response.getOutputStream(), RESPONSE_ENCODING));
         response.setContentType("text/xml");
         doSodaDescriptor(writer, request.getRequestURL().toString());
         writer.close();
      }



   protected void execRequest(HttpServletRequest request, HttpServletResponse response) 
         throws ServletException, IOException, UnsupportedEncodingException

      {
         long    startTime_msec = System.currentTimeMillis();
         long    startTime_nsec = System.nanoTime();

         ServletOutputStream  respOutputStream = response.getOutputStream();

         try
         {
            Map<String, String[]> params = request.getParameterMap();

            String id   = SingleStringParam.parseSingleStringParam(params, "ID");
            Pos    pos  = Pos.parsePos(params);
            Band   band = Band.parseBand(params);
            Time   time = Time.parseTime(params);
            Pol    pol  = Pol.parsePol(params);
            String pixels = SingleStringParam.parseSingleStringParam(params, "PIXELS");

            String respFormat = sodaReq_getResponseFormat(request, DEFAULT_RESPONSEFORMAT);

            LOGGER.finest("responseFormat: " + respFormat);

            if(respFormat.startsWith("application/fits"))
            {
               response.setContentType(respFormat);
               doCutoutStream(id, pos, band, time, pol, pixels, respOutputStream);
            }
            else if(respFormat.startsWith("application/x-vlkb+xml"))
            {
               boolean showDuration    = true;
               boolean countNullValues = vlkbReq_getNullValues(request);
               response.setContentType(respFormat);

               CutResult cutResult = doCutoutFile(id, pos, band, time, pol, pixels, countNullValues);// FIXME wh? , respFormat);


               /* FIXME errors from engine not checked - cut-file might not have been created */

               PrintWriter writer =new PrintWriter(new OutputStreamWriter(respOutputStream, RESPONSE_ENCODING));

               String accessUrl = convertLocalPathnameToRemoteUrl(cutResult.fileName,
                     settings.fitsPaths.cutouts(),
                     settings.fitsPaths.cutoutsUrl());

               XmlSerializer.serializeToLegacyCutResult(writer, RESPONSE_ENCODING,
                     cutResult, accessUrl,
                     id, pos, band, time, pol, pixels, countNullValues,
                     showDuration, startTime_msec);

               writer.close(); /* must close to force flush to complete the xml */
            }
            else
            {
               throw new IllegalArgumentException("Unsupported RESPONSEFORMAT value : " + respFormat);
            }

         }
         catch(MultiValuedParamNotSupported ex)
         {
            LOGGER.warning("MultiValuedParamNotSupported: " + ex.getMessage());

            response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
            response.setContentType("text/plain");
            PrintWriter writer = new PrintWriter(new OutputStreamWriter(respOutputStream, RESPONSE_ENCODING));

            Lib.doMultiValuedParamNotSupported(ex.getMessage(), writer);

            writer.close();
         }
         catch(IllegalArgumentException ex)
         {
            LOGGER.warning("IllegalArgumentException: " + ex.getMessage());

            response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
            response.setContentType("text/plain");
            PrintWriter writer = new PrintWriter(new OutputStreamWriter(respOutputStream, RESPONSE_ENCODING));

            Lib.doUsageError(ex.getMessage(), writer);

            writer.close();
         }
         catch(Exception ex)
         {
            LOGGER.severe("Exception: " + ex.getMessage());
            ex.printStackTrace();

            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
            response.setContentType("text/plain");
            PrintWriter writer = new PrintWriter(new OutputStreamWriter(respOutputStream, RESPONSE_ENCODING));

            Lib.doError(ex.toString(), writer);

            writer.close();
         }
         finally
         {
            respOutputStream.close();
         }

         LOGGER.fine("RUNTIME[sec]: "+String.valueOf( (System.nanoTime() - startTime_nsec) / 1000000000.0 ));
      }

   private String convertLocalPathnameToRemoteUrl(String localPathname,
         String FITScutpath, String FITSRemoteUrlCutouts)
   {
      LOGGER.fine("trace " + localPathname);
      String fileName = localPathname.replaceAll(FITScutpath + "/", "");
      LOGGER.finest("local filename: " + fileName);
      String remotefname = FITSRemoteUrlCutouts + "/" + fileName;
      LOGGER.finest("remote url    : " + remotefname);
      return remotefname;
   }






   /* SODA */


   /* return null if value not present or the value if present exactly once
    * else throw MultiplicityNotSupoorted SODA_error
    */ 
   private String soda_getSingleValue(HttpServletRequest req, String name)
   {
      String[] valArr = req.getParameterValues(name);

      if(valArr == null)
         return null;
      else
         if(valArr.length == 0)
            return null;
         else if(valArr.length == 1)
            return valArr[0];
         else
            throw new IllegalArgumentException(
                  "MultiValuedParamNotSupported: " + name + " was found " + valArr.length + " times");
   }


   private String sodaReq_getResponseFormat(HttpServletRequest req, String defaultResponseFormat)
   {
      String respFormat = soda_getSingleValue(req, "RESPONSEFORMAT");
      return ((respFormat == null) ? defaultResponseFormat : respFormat);
   }


   private boolean vlkbReq_getNullValues(HttpServletRequest request)
   {
      return (null != soda_getSingleValue(request, "nullvals"));
   }

}



/* from SODA (upon error):
   Error codes are specified in DALI. Error documents should be text
   using the text/plain content-type and the text must begin with one of the
   following strings:

   Error CodeDescription
   ---------------------------------------
Error: General error (not covered below)

AuthenticationError: Not authenticated
AuthorizationError: Not authorized to access the resource

ServiceUnavailable: Transient error (could succeed with retry)
UsageError: Permanent error (retry pointless)

MultiValuedParamNotSupported: request included multiple values for a parameter
but the service only supports a single value
*/


/* from DALI (upon successful request):
   The service should set HTTP headers (Fielding and Gettys et al.,
   1999) that are useful to the correct values where possible. Recommended
   headers to set when possible:
   Content-Type
   Content-Encoding
   Content-Length  -- not in SPDA-stream impossible to know
   Last-Modified   -- not in SODA-stream impossible to know
   */