import java.util.logging.Logger; import java.util.logging.Level; import java.util.List; import java.util.ArrayList; import java.util.Arrays; import java.time.Instant; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileNotFoundException; import java.nio.file.StandardOpenOption; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.*;// Timestamp in cut-filename import java.io.ByteArrayOutputStream; // for SODA direct streaming doSubimgStream // tar.gz compress import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; import java.io.BufferedOutputStream; import java.util.Date; import java.text.SimpleDateFormat; import org.json.simple.JSONArray; import vo.parameter.*; import vo.error.*; class VlkbCli implements Vlkb { static final Logger LOGGER = Logger.getLogger(VlkbCli.class.getName()); private Settings settings = null; private Subsurvey[] subsurveys = null; private Resolver resolver = null; private Soda soda = null; public VlkbCli() { LOGGER.fine("trace VlkbCli()"); this.settings = Settings.getInstance(); this.soda = new SodaImpl(settings.fitsPaths); this.resolver = (settings.dbConn.isDbUriEmpty() ? new ResolverFromId(subsurveys) : new ResolverByObsCore(settings.dbConn, subsurveys)); } public VlkbCli(Settings settings) { LOGGER.fine("trace VlkbCli(settings)"); this.settings = settings; this.soda = new SodaImpl(settings.fitsPaths); this.resolver = (settings.dbConn.isDbUriEmpty() ? new ResolverFromId(subsurveys) : new ResolverByObsCore(settings.dbConn, subsurveys)); } public VlkbCli(Settings settings, Subsurvey[] subsurveys) { LOGGER.fine("trace VlkbCli(settings, subsurveys)"); this.settings = settings; this.subsurveys = subsurveys; this.soda = new SodaImpl(settings.fitsPaths); this.resolver = new ResolverFromId(subsurveys); //this.resolver = (settings.dbConn.isDbUriEmpty() ? new ResolverFromId(subsurveys) // : new ResolverByObsCore(settings.dbConn, subsurveys)); } public CutResult doMerge(String[] idArr, Coord coord, boolean countNullValues) throws FileNotFoundException, IOException { LOGGER.fine("trace doMerge by CLI is NOT IMPLEMENTED (only by AMQP)"); return new CutResult(); } public CutResult doFile(String relPathname, int hdunum, Pos pos, Band band, Time time, Pol pol, String pixels, boolean countNullValues, FitsCard[] extraCards) throws IOException, InterruptedException { LOGGER.fine("trace: " + pos.toString() ); CutResult cutResult = new CutResult(); LOGGER.finer("Using doStream() to local file"); String absSubimgPathname = settings.fitsPaths.cutouts() + "/" + generateSubimgPathname(relPathname, hdunum); LOGGER.finer("Uses local filename : " + absSubimgPathname); OutputStream fileOutputStream = new FileOutputStream( new File(absSubimgPathname) ); soda.doStream(relPathname, hdunum, pos, band, time, pol, pixels, fileOutputStream); // engine returns absPathname see common/cutout.cpp::do_cutout_file() cutResult.fileName = absSubimgPathname; cutResult.fileSize = Files.size(Paths.get(absSubimgPathname)); if(countNullValues) { cutResult.nullValueCount = doCountNullValues(absSubimgPathname, 1); } if(extraCards == null || (extraCards.length < 1)) { LOGGER.finer("Adding extraCards to cut-file implemented only in VlkbAmql"); } cutResult.pixels = null; return cutResult; } private NullValueCount doCountNullValues(String absPathname, int hdunum) throws IOException, InterruptedException { ByteArrayOutputStream bos = new ByteArrayOutputStream(); if(bos == null) throw new AssertionError("byte output stream for bounds was not created, is null"); String[] cmdBounds = new String[3]; cmdBounds[0] = "/usr/local/bin/vlkb"; cmdBounds[1] = "nullvals"; cmdBounds[2] = absPathname; ExecCmd exec = new ExecCmd(); exec.doRun(bos, cmdBounds); LOGGER.finest("exec NullVals exitValue: " + exec.exitValue); bos.close(); boolean hasResult = (exec.exitValue == 0); if(hasResult) { String nullValsString = new String(bos.toByteArray()); LOGGER.finest("vlkb nullvals: " + nullValsString); if((nullValsString != null) && nullValsString.trim().isEmpty()) { throw new AssertionError("'vlkb nullvals' returned empty string"); } // parse result: '<fill-ratio> <nullvals-count> <tot-count>' String[] splitStr = nullValsString.trim().split("\\s+"); if(splitStr.length != 3) throw new AssertionError("'vlkb nullvals' did not return 3 numbers but: " + nullValsString); NullValueCount nvc = new NullValueCount(); nvc.percent = Double.parseDouble(splitStr[0]); nvc.nullCount = Long.parseLong(splitStr[1]); nvc.totalCount = Long.parseLong(splitStr[2]); return nvc; } else { throw new AssertionError("'vlkb nullvals' exited without results for: " + absPathname); } } private CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels, boolean countNullValues/*, Subsurvey[] subsurveys*/) throws IOException, InterruptedException { LOGGER.fine("trace"); FitsCard[] extraCards = null; this.resolver.resolve(id); String relPathname = this.resolver.relPathname(); int hdunum = this.resolver.hdunum(); String subsurveyId = this.resolver.obsCollection(); if(subsurveyId != null) { extraCards = Subsurvey.subsurveysFindCards(this.subsurveys, subsurveyId); } else { LOGGER.finer("Resolver returns subsurveyId null: no extraCards loaded."); } final String DEFAULT_TIME_SYSTEM = "MJD_UTC"; // FIXME take from confif file CutResult cutResult = doFile(relPathname, hdunum, pos, band, time, pol, pixels, countNullValues, extraCards); return cutResult; } private String generateSubimgPathname(String relPathname, int hdunum) { String cutfitsname = "vlkb-cutout"; Instant instant = Instant.now() ; String timestamp = instant.toString().replace(":","-").replace(".","_"); String tempPathname1 = relPathname.replaceAll("/","-"); String tempPathname2 = tempPathname1.replaceAll(" ","_"); if(hdunum == 1) { return cutfitsname + "_" + timestamp + "_" + tempPathname2; } else { String extnum = "EXT" + String.valueOf(hdunum-1); return cutfitsname + "_" + timestamp + "_" + extnum + "_" + tempPathname2; } } public MCutResult doMCutout(String jdlJson) throws IOException, InterruptedException { LOGGER.fine("trace"); MCutResult mCutResult; CutArgs[] cutArgsArr = CutArgs.parseCutArgsArr(jdlJson); MCutResult.Cut[] cutResultArr = doCutouts( cutArgsArr ); String respJsonString = genResponseJson( cutResultArr ); mCutResult = doCompressCutFiles( cutResultArr, respJsonString ); return mCutResult; } private MCutResult.Cut[] doCutouts(CutArgs[] cutArgsArr) { LOGGER.fine("trace, count of cuts : " + String.valueOf( cutArgsArr.length ) ); List<MCutResult.Cut> cutResList = new ArrayList<MCutResult.Cut>(); int ix = 0; for(CutArgs cutArgs: cutArgsArr) { MCutResult.Cut cut = doFileByIdWithErr(cutArgs.id, cutArgs.pos, cutArgs.band, cutArgs.time, cutArgs.pol, cutArgs.pixels, cutArgs.countNullValues);//, null);//cutArgs.extraCards); cut.index = ix++; LOGGER.finest("cut" + String.valueOf(cut.index) + " : " + cut.content); cutResList.add(cut); } return cutResList.toArray(new MCutResult.Cut[0]); } // FIXME implement similar for Merge: MCutResult = call-Montage-demosaic-sequence(cutResultArr) private MCutResult doCompressCutFiles(MCutResult.Cut[] cutArr, String respJsonString) throws IOException, InterruptedException { Instant instant = Instant.now(); String timestamp = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + "_" + instant.getNano(); final String tgzFileName = settings.fitsPaths.cutouts() + "/mcutout_" + timestamp + ".tar.gz"; // generate response-*.json with timestamp String respJsonPathname = settings.fitsPaths.cutouts() + "/response_" + timestamp + ".json"; try (PrintWriter out = new PrintWriter(respJsonPathname)) { out.print(respJsonString); } List<Path> paths = new ArrayList<Path>(); paths.add(Paths.get(respJsonPathname)); for(MCutResult.Cut cut : cutArr) { LOGGER.finest("cut-id"+ String.valueOf(cut.index) + " -> " + cut.content); if(cut.contentType == MCutResult.Cut.ContentType.FILENAME) { Path p = Paths.get(cut.content); paths.add(p); } } Path output = Paths.get(tgzFileName); createTarGzipFiles(paths, output); MCutResult mCutResult = new MCutResult(); mCutResult.cutResArr = cutArr; mCutResult.fileName = tgzFileName; mCutResult.resJsonPathname = respJsonPathname; mCutResult.fileSize = Files.size(output); return mCutResult; } private String genResponseJson(MCutResult.Cut[] cutArr) { List<Path> paths = new ArrayList<Path>(); JSONArray jArr = new JSONArray(); for(MCutResult.Cut cut : cutArr) { jArr.add(cut.toJsonObject()); } return jArr.toString(); } private MCutResult.Cut doFileByIdWithErr(String id, Pos pos, Band band, Time time, Pol pol, String pixels, boolean countNullValues/*, Subsurvey[] subsurveys*/) { LOGGER.fine("trace"); MCutResult mCutResult = new MCutResult(); MCutResult.Cut cut = mCutResult.new Cut(/* FIXME eventually add here new Inputs(id, pos,..) */); try { CutResult cutResult = doFileById(id, pos, band, time, pol, pixels, countNullValues/*, subsurveys*/); cut.content = cutResult.fileName; cut.contentType = MCutResult.Cut.ContentType.FILENAME; } catch(MultiValuedParamNotSupported ex) { cut.content = "MultiValuedParamNotSupported: " + ex.getMessage(); cut.contentType = MCutResult.Cut.ContentType.BAD_REQUEST; LOGGER.warning(cut.content); } catch(IllegalArgumentException ex) { cut.content = "IllegalArgumentException: " + ex.getMessage(); cut.contentType = MCutResult.Cut.ContentType.BAD_REQUEST; LOGGER.warning(cut.content); } catch(Exception ex) { cut.content = "Exception: " + ex.getMessage(); cut.contentType = MCutResult.Cut.ContentType.SERVICE_ERROR; LOGGER.severe(cut.content); ex.printStackTrace(); } return cut; } /* <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-compress</artifactId> <version>1.20</version> </dependency> import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; */ private static void createTarGzipFiles(List<Path> paths, Path output) throws IOException { try (OutputStream fOut = Files.newOutputStream(output); BufferedOutputStream buffOut = new BufferedOutputStream(fOut); GzipCompressorOutputStream gzOut = new GzipCompressorOutputStream(buffOut); TarArchiveOutputStream tOut = new TarArchiveOutputStream(gzOut)) { for (Path path : paths) { if (!Files.isRegularFile(path)) { throw new IOException("Must be regular file, but was : " + path.toString()); } TarArchiveEntry tarEntry = new TarArchiveEntry( path.toFile(), path.getFileName().toString()); tOut.putArchiveEntry(tarEntry); // copy file to TarArchiveOutputStream Files.copy(path, tOut); tOut.closeArchiveEntry(); } tOut.finish(); } } }