From 30e19f826da18faa2a215d73a1d622cc3fe85be3 Mon Sep 17 00:00:00 2001
From: Robert Butora <robert.butora@inaf.it>
Date: Thu, 16 May 2024 16:01:05 +0200
Subject: [PATCH] moves doFile doMCutout from CutoutImpl -> DatasetsCli
DatasetsAmqp
---
.../servlet/src/main/java/cutout/Cutout.java | 11 -
.../src/main/java/cutout/CutoutImpl.java | 190 ------------------
.../src/main/java/cutout/Datasets.java | 17 +-
.../src/main/java/cutout/DatasetsAmqp.java | 94 +++++++++
.../src/main/java/cutout/DatasetsCli.java | 182 ++++++++++++++++-
.../src/main/java/webapi/ServletCutout.java | 5 +-
6 files changed, 289 insertions(+), 210 deletions(-)
diff --git a/data-access/servlet/src/main/java/cutout/Cutout.java b/data-access/servlet/src/main/java/cutout/Cutout.java
index 0695e85..1ab96ad 100644
--- a/data-access/servlet/src/main/java/cutout/Cutout.java
+++ b/data-access/servlet/src/main/java/cutout/Cutout.java
@@ -18,16 +18,5 @@ public interface Cutout
Pos pos, Band band, Time time, Pol pol, String pixels,
OutputStream outputStream) throws IOException, InterruptedException;
-
- public CutResult doFile(String relPathname, int hdunum,
- Pos pos, Band band, Time time, Pol pol, String pixels,
- boolean countNullValues, FitsCard[] extraCards)
- throws IOException, InterruptedException;
-
-
- public CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
- boolean countNullValues, Subsurvey[] subsurveys)
- throws IOException, InterruptedException;
-
}
diff --git a/data-access/servlet/src/main/java/cutout/CutoutImpl.java b/data-access/servlet/src/main/java/cutout/CutoutImpl.java
index e49c1c8..3660f16 100644
--- a/data-access/servlet/src/main/java/cutout/CutoutImpl.java
+++ b/data-access/servlet/src/main/java/cutout/CutoutImpl.java
@@ -276,195 +276,5 @@ class CutoutImpl implements Cutout
return other.split("\\s+");
}
-
- private NullValueCount doCountNullValues(String absPathname, int hdunum)
- throws IOException, InterruptedException
- {
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- if(bos == null)
- throw new AssertionError("byte output stream for bounds was not created, is null");
-
- String[] cmdBounds = new String[3];
- cmdBounds[0] = "/usr/local/bin/vlkb";
- cmdBounds[1] = "nullvals";
- cmdBounds[2] = absPathname;
-
- ExecCmd exec = new ExecCmd();
- exec.doRun(bos, cmdBounds);
- LOGGER.info("exec NullVals exitValue: " + exec.exitValue);
-
- bos.close();
-
- boolean hasResult = (exec.exitValue == 0);
- if(hasResult)
- {
- String nullValsString = new String(bos.toByteArray());
- LOGGER.info("vlkb nullvals: " + nullValsString);
-
- if((nullValsString != null) && nullValsString.trim().isEmpty())
- {
- throw new AssertionError("'vlkb nullvals' returned empty string");
- }
-
- // parse result: '<fill-ratio> <nullvals-count> <tot-count>'
-
- String[] splitStr = nullValsString.trim().split("\\s+");
- if(splitStr.length != 3) throw new AssertionError("'vlkb nullvals' did not return 3 numbers but: " + nullValsString);
-
- NullValueCount nvc = new NullValueCount();
- nvc.percent = Double.parseDouble(splitStr[0]);
- nvc.nullCount = Long.parseLong(splitStr[1]);
- nvc.totalCount = Long.parseLong(splitStr[2]);
- return nvc;
- }
- else
- {
- throw new AssertionError("'vlkb nullvals' exited without results for: " + absPathname);
- }
- }
-
-
-
- public CutResult doFile(String relPathname, int hdunum,
- Pos pos, Band band, Time time, Pol pol, String pixels,
- boolean countNullValues, FitsCard[] extraCards)
- throws IOException, InterruptedException
- {
- LOGGER.info("trace: " + pos.toString() );
-
- CutResult cutResult = new CutResult();
-
- if(settings.amqpConn.isHostnameEmpty())
- {
- LOGGER.info("Using doStream() to local file");
-
- String absSubimgPathname = settings.fitsPaths.cutouts()
- + "/" + generateSubimgPathname(relPathname, hdunum);
-
- LOGGER.info("Uses local filename : " + absSubimgPathname);
-
- OutputStream fileOutputStream = new FileOutputStream( new File(absSubimgPathname) );
-
- doStream(relPathname, hdunum, pos, band, time, pol, pixels, fileOutputStream);
-
- // engine returns absPathname see common/cutout.cpp::do_cutout_file()
- cutResult.fileName = absSubimgPathname;
- cutResult.fileSize = Files.size(Paths.get(absSubimgPathname));
-
- if(countNullValues)
- {
- cutResult.nullValueCount = doCountNullValues(absSubimgPathname, 1);
- }
-
- if(extraCards == null || (extraCards.length < 1))
- {
- LOGGER.info("Adding extraCards to cut-file not implemented when using 'vlkb' exec (implemented in engine vlkbd/AMQP)");
- }
-
- cutResult.pixels = null;
- }
- else
- {
- LOGGER.info("Using AMQP");
-
- JsonEncoder jReq = new JsonEncoder();
- jReq.add(relPathname, hdunum);
- jReq.add(pos);
- jReq.add(band);
- jReq.add(time);
- jReq.add(pol);
-
- // jReq.add(pixels), FIXME implement to supoort PIXLES in vlkb-legacy by AMQP
-
- jReq.add(countNullValues);
- jReq.add(extraCards);
-
- String outJson = RpcOverAmqp.doRpc( settings.amqpConn, jReq.toString() );
-
- cutResult = JsonDecoder.responseFromCutoutJson( outJson );
- }
-
- return cutResult;
- }
-
-
-
- public CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
- boolean countNullValues, Subsurvey[] subsurveys)
- throws IOException, InterruptedException
- {
- LOGGER.info("trace");
-
- String relPathname;
- int hdunum;
-
- FitsCard[] extraCards = null;
-
- String dbUri = settings.dbConn.uri();
-
- if(settings.dbConn.isDbUriEmpty())
- {
- Resolver rsl = new ResolverFromId();
- rsl.resolve(id);
- relPathname = rsl.relPathname();
- hdunum = rsl.hdunum();
-
- /* FIXME needs also match on filename - some subsurveys have the same storage-path,
- * and file-filter distiguishes frequences
- * OR
- * ivoid must include obs-collection
-
- Path path = Paths.get(rsl.relPathname());
- String storagePath = path.getParent().toString();
- extraCards = Subsurvey.subsurveysFindCardsByStoragePath(subsurveys, storagePath);
- */
- }
- else
- {
- ResolverByObsCore rsl = new ResolverByObsCore(settings.dbConn, subsurveys);
- rsl.resolve(id);
- relPathname = rsl.relPathname();
- hdunum = rsl.hdunum();
- String subsurveyId = rsl.obsCollection();
- if(subsurveyId != null)
- {
- extraCards = Subsurvey.subsurveysFindCards(subsurveys, subsurveyId);
- }
- else
- {
- LOGGER.info("Resolver with Obscore returns subsurveyId null: no extraCards loaded.");
- }
- }
-
- final String DEFAULT_TIME_SYSTEM = "MJD_UTC"; // FIXME take from confif file
-
- CutResult cutResult = doFile(relPathname, hdunum, pos, band, time, pol, pixels,
- countNullValues, extraCards);
-
- return cutResult;
- }
-
-
- private String generateSubimgPathname(String relPathname, int hdunum)
- {
- String cutfitsname = "vlkb-cutout";
-
- Instant instant = Instant.now() ;
- String timestamp = instant.toString().replace(":","-").replace(".","_");
-
- String tempPathname1 = relPathname.replaceAll("/","-");
- String tempPathname2 = tempPathname1.replaceAll(" ","_");
-
- if(hdunum == 1)
- {
- return cutfitsname + "_" + timestamp + "_" + tempPathname2;
- }
- else
- {
- String extnum = "EXT" + String.valueOf(hdunum-1);
- return cutfitsname + "_" + timestamp + "_" + extnum + "_" + tempPathname2;
- }
- }
-
}
diff --git a/data-access/servlet/src/main/java/cutout/Datasets.java b/data-access/servlet/src/main/java/cutout/Datasets.java
index d07cdc0..7df294e 100644
--- a/data-access/servlet/src/main/java/cutout/Datasets.java
+++ b/data-access/servlet/src/main/java/cutout/Datasets.java
@@ -9,15 +9,28 @@ import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.time.Instant;//Timestamp in cut-filename
+import vo.parameter.*;
+
public interface Datasets
{
+ public CutResult doFile(String relPathname, int hdunum,
+ Pos pos, Band band, Time time, Pol pol, String pixels,
+ boolean countNullValues, FitsCard[] extraCards)
+ throws IOException, InterruptedException;
+
+
+ public CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
+ boolean countNullValues, Subsurvey[] subsurveys)
+ throws IOException, InterruptedException;
- public CutResult doMerge(String[] idArr, Coord coord, boolean countNullValues)
- throws FileNotFoundException, IOException;
public MCutResult doMCutout(String jdlJson)
throws IOException, InterruptedException;
+
+
+ public CutResult doMerge(String[] idArr, Coord coord, boolean countNullValues)
+ throws FileNotFoundException, IOException;
}
diff --git a/data-access/servlet/src/main/java/cutout/DatasetsAmqp.java b/data-access/servlet/src/main/java/cutout/DatasetsAmqp.java
index 1fd7786..95501ef 100644
--- a/data-access/servlet/src/main/java/cutout/DatasetsAmqp.java
+++ b/data-access/servlet/src/main/java/cutout/DatasetsAmqp.java
@@ -67,6 +67,100 @@ class DatasetsAmqp implements Datasets
}
+///////////////////////////////////////////////////////////////////////////////////
+
+ public CutResult doFile(String relPathname, int hdunum,
+ Pos pos, Band band, Time time, Pol pol, String pixels,
+ boolean countNullValues, FitsCard[] extraCards)
+ throws IOException, InterruptedException
+ {
+ LOGGER.info("trace: " + pos.toString() );
+
+ CutResult cutResult = new CutResult();
+
+ LOGGER.info("Using AMQP");
+
+ JsonEncoder jReq = new JsonEncoder();
+ jReq.add(relPathname, hdunum);
+ jReq.add(pos);
+ jReq.add(band);
+ jReq.add(time);
+ jReq.add(pol);
+
+ // jReq.add(pixels), FIXME implement to supoort PIXLES in vlkb-legacy by AMQP
+
+ jReq.add(countNullValues);
+ jReq.add(extraCards);
+
+ String outJson = RpcOverAmqp.doRpc( settings.amqpConn, jReq.toString() );
+
+ cutResult = JsonDecoder.responseFromCutoutJson( outJson );
+
+ return cutResult;
+ }
+
+
+
+ public CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
+ boolean countNullValues, Subsurvey[] subsurveys)
+ throws IOException, InterruptedException
+ {
+ LOGGER.info("trace");
+
+ String relPathname;
+ int hdunum;
+
+ FitsCard[] extraCards = null;
+
+ String dbUri = settings.dbConn.uri();
+
+ if(settings.dbConn.isDbUriEmpty())
+ {
+ Resolver rsl = new ResolverFromId();
+ rsl.resolve(id);
+ relPathname = rsl.relPathname();
+ hdunum = rsl.hdunum();
+
+ /* FIXME needs also match on filename - some subsurveys have the same storage-path,
+ * and file-filter distiguishes frequences
+ * OR
+ * ivoid must include obs-collection
+
+ Path path = Paths.get(rsl.relPathname());
+ String storagePath = path.getParent().toString();
+ extraCards = Subsurvey.subsurveysFindCardsByStoragePath(subsurveys, storagePath);
+ */
+ }
+ else
+ {
+ ResolverByObsCore rsl = new ResolverByObsCore(settings.dbConn, subsurveys);
+ rsl.resolve(id);
+ relPathname = rsl.relPathname();
+ hdunum = rsl.hdunum();
+ String subsurveyId = rsl.obsCollection();
+ if(subsurveyId != null)
+ {
+ extraCards = Subsurvey.subsurveysFindCards(subsurveys, subsurveyId);
+ }
+ else
+ {
+ LOGGER.info("Resolver with Obscore returns subsurveyId null: no extraCards loaded.");
+ }
+ }
+
+ final String DEFAULT_TIME_SYSTEM = "MJD_UTC"; // FIXME take from confif file
+
+ CutResult cutResult = doFile(relPathname, hdunum, pos, band, time, pol, pixels,
+ countNullValues, extraCards);
+
+ return cutResult;
+ }
+
+
+
+
+ ///////////////////////////////////////////////////////////////////////////////////
+
public MCutResult doMCutout(String jdlJson)
throws IOException
diff --git a/data-access/servlet/src/main/java/cutout/DatasetsCli.java b/data-access/servlet/src/main/java/cutout/DatasetsCli.java
index a8fc5d9..84bdf05 100644
--- a/data-access/servlet/src/main/java/cutout/DatasetsCli.java
+++ b/data-access/servlet/src/main/java/cutout/DatasetsCli.java
@@ -16,6 +16,7 @@ import java.io.PrintWriter;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.FileNotFoundException;
import java.nio.file.StandardOpenOption;
import java.nio.file.Files;
@@ -81,8 +82,179 @@ class DatasetsCli implements Datasets
+///////////////////////////////////////////////////////////////////////////////////////
+
+
+ private NullValueCount doCountNullValues(String absPathname, int hdunum)
+ throws IOException, InterruptedException
+ {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ if(bos == null)
+ throw new AssertionError("byte output stream for bounds was not created, is null");
+
+ String[] cmdBounds = new String[3];
+ cmdBounds[0] = "/usr/local/bin/vlkb";
+ cmdBounds[1] = "nullvals";
+ cmdBounds[2] = absPathname;
+
+ ExecCmd exec = new ExecCmd();
+ exec.doRun(bos, cmdBounds);
+ LOGGER.info("exec NullVals exitValue: " + exec.exitValue);
+
+ bos.close();
+
+ boolean hasResult = (exec.exitValue == 0);
+ if(hasResult)
+ {
+ String nullValsString = new String(bos.toByteArray());
+ LOGGER.info("vlkb nullvals: " + nullValsString);
+
+ if((nullValsString != null) && nullValsString.trim().isEmpty())
+ {
+ throw new AssertionError("'vlkb nullvals' returned empty string");
+ }
+
+ // parse result: '<fill-ratio> <nullvals-count> <tot-count>'
+
+ String[] splitStr = nullValsString.trim().split("\\s+");
+ if(splitStr.length != 3) throw new AssertionError("'vlkb nullvals' did not return 3 numbers but: " + nullValsString);
+
+ NullValueCount nvc = new NullValueCount();
+ nvc.percent = Double.parseDouble(splitStr[0]);
+ nvc.nullCount = Long.parseLong(splitStr[1]);
+ nvc.totalCount = Long.parseLong(splitStr[2]);
+ return nvc;
+ }
+ else
+ {
+ throw new AssertionError("'vlkb nullvals' exited without results for: " + absPathname);
+ }
+ }
+
+
+ public CutResult doFile(String relPathname, int hdunum,
+ Pos pos, Band band, Time time, Pol pol, String pixels,
+ boolean countNullValues, FitsCard[] extraCards)
+ throws IOException, InterruptedException
+ {
+ LOGGER.info("trace: " + pos.toString() );
+
+ CutResult cutResult = new CutResult();
+
+ LOGGER.info("Using doStream() to local file");
+
+ String absSubimgPathname = settings.fitsPaths.cutouts()
+ + "/" + generateSubimgPathname(relPathname, hdunum);
+
+ LOGGER.info("Uses local filename : " + absSubimgPathname);
+
+ OutputStream fileOutputStream = new FileOutputStream( new File(absSubimgPathname) );
+
+ cutout.doStream(relPathname, hdunum, pos, band, time, pol, pixels, fileOutputStream);
+
+ // engine returns absPathname see common/cutout.cpp::do_cutout_file()
+ cutResult.fileName = absSubimgPathname;
+ cutResult.fileSize = Files.size(Paths.get(absSubimgPathname));
+
+ if(countNullValues)
+ {
+ cutResult.nullValueCount = doCountNullValues(absSubimgPathname, 1);
+ }
+
+ if(extraCards == null || (extraCards.length < 1))
+ {
+ LOGGER.info("Adding extraCards to cut-file not implemented when using 'vlkb' exec (implemented in engine vlkbd/AMQP)");
+ }
+
+ cutResult.pixels = null;
+ return cutResult;
+ }
+
+
+ public CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
+ boolean countNullValues, Subsurvey[] subsurveys)
+ throws IOException, InterruptedException
+ {
+ LOGGER.info("trace");
+
+ String relPathname;
+ int hdunum;
+
+ FitsCard[] extraCards = null;
+
+ String dbUri = settings.dbConn.uri();
+
+ if(settings.dbConn.isDbUriEmpty())
+ {
+ Resolver rsl = new ResolverFromId();
+ rsl.resolve(id);
+ relPathname = rsl.relPathname();
+ hdunum = rsl.hdunum();
+
+ /* FIXME needs also match on filename - some subsurveys have the same storage-path,
+ * and file-filter distiguishes frequences
+ * OR
+ * ivoid must include obs-collection
+
+ Path path = Paths.get(rsl.relPathname());
+ String storagePath = path.getParent().toString();
+ extraCards = Subsurvey.subsurveysFindCardsByStoragePath(subsurveys, storagePath);
+ */
+ }
+ else
+ {
+ ResolverByObsCore rsl = new ResolverByObsCore(settings.dbConn, subsurveys);
+ rsl.resolve(id);
+ relPathname = rsl.relPathname();
+ hdunum = rsl.hdunum();
+ String subsurveyId = rsl.obsCollection();
+ if(subsurveyId != null)
+ {
+ extraCards = Subsurvey.subsurveysFindCards(subsurveys, subsurveyId);
+ }
+ else
+ {
+ LOGGER.info("Resolver with Obscore returns subsurveyId null: no extraCards loaded.");
+ }
+ }
+
+ final String DEFAULT_TIME_SYSTEM = "MJD_UTC"; // FIXME take from confif file
+
+ CutResult cutResult = doFile(relPathname, hdunum, pos, band, time, pol, pixels,
+ countNullValues, extraCards);
+
+ return cutResult;
+ }
+
+ private String generateSubimgPathname(String relPathname, int hdunum)
+ {
+ String cutfitsname = "vlkb-cutout";
+
+ Instant instant = Instant.now() ;
+ String timestamp = instant.toString().replace(":","-").replace(".","_");
+
+ String tempPathname1 = relPathname.replaceAll("/","-");
+ String tempPathname2 = tempPathname1.replaceAll(" ","_");
+
+ if(hdunum == 1)
+ {
+ return cutfitsname + "_" + timestamp + "_" + tempPathname2;
+ }
+ else
+ {
+ String extnum = "EXT" + String.valueOf(hdunum-1);
+ return cutfitsname + "_" + timestamp + "_" + extnum + "_" + tempPathname2;
+ }
+ }
+
+
+ ///////////////////////////////////////////////////////////////////////////////////////
+
+
+
+
public MCutResult doMCutout(String jdlJson)
- throws IOException, InterruptedException
+ throws IOException, InterruptedException
{
LOGGER.info("trace");
@@ -128,7 +300,7 @@ class DatasetsCli implements Datasets
{
Instant instant = Instant.now();
String timestamp = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + "_" + instant.getNano();
-
+
final String tgzFileName = settings.fitsPaths.cutouts() + "/mcutout_" + timestamp + ".tar.gz";
// generate response-*.json with timestamp
@@ -190,7 +362,7 @@ class DatasetsCli implements Datasets
try
{
- CutResult cutResult = cutout.doFileById(id,
+ CutResult cutResult = doFileById(id,
pos, band, time, pol, pixels,
countNullValues, subsurveys);
@@ -222,7 +394,7 @@ class DatasetsCli implements Datasets
-/*
+ /*
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
@@ -232,7 +404,7 @@ class DatasetsCli implements Datasets
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
-*/
+ */
private static void createTarGzipFiles(List<Path> paths, Path output)
throws IOException
{
diff --git a/data-access/servlet/src/main/java/webapi/ServletCutout.java b/data-access/servlet/src/main/java/webapi/ServletCutout.java
index 7ac6c45..ca49a16 100644
--- a/data-access/servlet/src/main/java/webapi/ServletCutout.java
+++ b/data-access/servlet/src/main/java/webapi/ServletCutout.java
@@ -59,7 +59,8 @@ public class ServletCutout extends javax.servlet.http.HttpServlet
private Subsurvey[] subsurveys = null;
- protected Cutout cutout = new CutoutImpl(settings, subsurveys);
+ protected Cutout cutout = new CutoutImpl(settings, subsurveys);
+ protected Datasets datasets = ( settings.amqpConn.isHostnameEmpty() ? new DatasetsCli(settings): new DatasetsAmqp(settings) );
public void init() throws ServletException
{
@@ -135,7 +136,7 @@ public class ServletCutout extends javax.servlet.http.HttpServlet
{
LOGGER.info("trace");
- return cutout.doFileById(id, pos, band, time, pol, pixels, countNullValues, subsurveys);
+ return datasets.doFileById(id, pos, band, time, pol, pixels, countNullValues, subsurveys);
}
--
GitLab