From f09cc3427b319f71130c68806d5e130c293c45df Mon Sep 17 00:00:00 2001
From: David Tosolini <david.tosolini@inaf.it>
Date: Wed, 22 Jan 2020 13:22:07 +0100
Subject: [PATCH] Update checkfits.sh E' stato aggiunto del codice per usare
uno script di astropy-pyfits chimato fitscheck. La modifica scrive in tutti i
nuovi files.fits le keywords CHECKSUM e DATASUM
---
script/checkfits.sh | 577 +++++++++++++++++++++++++++-----------------
1 file changed, 361 insertions(+), 216 deletions(-)
diff --git a/script/checkfits.sh b/script/checkfits.sh
index 2fbcb5a..db69363 100755
--- a/script/checkfits.sh
+++ b/script/checkfits.sh
@@ -2,69 +2,103 @@
#: Title : checkfits.sh
#: Date : 2014/10/28
-#: Author : "Marco De Marco" <demarco@oats.inaf.it>
-#: Version : 1.0
+#: Author :"David Tosolini" david.tosolini@inaf.it /"Marco De Marco"
+#: Version : 2.0
#: Description : Fits verification and proccessing script
+# ===========================================================================
+
#Tools paths
+
+# --------------------------------------------------------------------------
+
VERIFY_TOOL="/usr/local/bin/fitsverify"
LISTHEAD_TOOL="/usr/local/bin/listhead"
MODHEAD_TOOL="/usr/local/bin/modhead"
+FITSCHECK_TOOL="/usr/bin/fitscheck"
+
+# ===========================================================================
if [ "$1" == "CHECK" ]; then
- #: Section : CHECK
- #: Parameter : none
- #: Response : CHECK OK
- #: : CHECK FATAL
- #: Description : Check availability of script tools
+# --------------------------------------------------------------------------
+ #: Section : CHECK
+ #: Parameter : none
+ #: Response : CHECK OK
+ #: : CHECK FATAL
+ #: Description : Check availability of script tools
+# ===========================================================================
+
+ # 1. Check fitsverify tools
+
+ CHECK_STRING="conform to the FITS format"
+
+ res=$($VERIFY_TOOL 2>&1)
+
+ check=$(echo $res | grep "$CHECK_STRING" | wc | awk '{print $1}')
+ if [ "$check" -lt "1" ]; then
+ echo "CHECK FATAL"
+ exit 0
+ fi
+
+
+ # 2. Check listhead tools
+
+ CHECK_STRING="Usage: listhead filename"
+
+ res=$($LISTHEAD_TOOL 2>&1)
+
+ check=$(echo $res | grep "$CHECK_STRING" | wc | awk '{print $1}')
+ if [ "$check" -lt "1" ]; then
+ echo "CHECK FATAL"
+ exit 0
+ fi
+
- #Check fitsverify tools
- CHECK_STRING="conform to the FITS format"
+ # 3. Check modhead tools
- res=$($VERIFY_TOOL 2>&1)
+ CHECK_STRING="Usage: modhead filename"
- check=$(echo $res | grep "$CHECK_STRING" | wc | awk '{print $1}')
- if [ "$check" -lt "1" ]; then
- echo "CHECK FATAL"
- exit 0
- fi
+ res=$($MODHEAD_TOOL 2>&1)
+
+ check=$(echo $res | grep "$CHECK_STRING" | wc | awk '{print $1}')
+ if [ "$check" -lt "1" ]; then
+ echo "CHECK FATAL"
+ exit 0
+ fi
- #Check listhead tools
- CHECK_STRING="Usage: listhead filename"
- res=$($LISTHEAD_TOOL 2>&1)
+ # 4. Check fitscheck tools
+ CHECK_STRING="Usage: fitscheck"
- check=$(echo $res | grep "$CHECK_STRING" | wc | awk '{print $1}')
- if [ "$check" -lt "1" ]; then
- echo "CHECK FATAL"
- exit 0
- fi
+ res=$($FITSCHECK_TOOL 2>&1)
- #Check modhead tools
- CHECK_STRING="Usage: modhead filename"
+ check=$(echo $res | grep "$CHECK_STRING" | wc | awk '{print $1}')
+ if [ "$check" -lt "1" ]; then
+ echo "CHECK FATAL"
+ exit 0
+ fi
- res=$($MODHEAD_TOOL 2>&1)
+ # if 1.2.3.4. are ok then the Section CHECK gives "CHECK OK" output
- check=$(echo $res | grep "$CHECK_STRING" | wc | awk '{print $1}')
- if [ "$check" -lt "1" ]; then
- echo "CHECK FATAL"
- exit 0
- fi
+ echo "CHECK OK"
+ exit 0
- echo "CHECK OK"
- exit 0
+# ===========================================================================
elif [ "$1" == "VALID" ]; then
- #: Section : VALID
- #: Parameter : file path
- #: Response : VALID OK
- #: : VALID IGNORE
- #: Description : Check file name compliance
+# ---------------------------------------------------------------------------
+ #: Section : VALID
+ #: Parameter : file path
+ #: Response : VALID OK
+ #: : VALID IGNORE
+ #: Description : Check file name compliance
+# ===========================================================================
- file=$2
- file_name=${file##*/}
+
+ file=$2
+ file_name=${file##*/}
#Check regex for tec files -> ignore
if [[ "${file_name,,}" =~ ^.*lbc.?tec.*\.(fits|fit|fts).*$ ]]; then
@@ -72,207 +106,318 @@ elif [ "$1" == "VALID" ]; then
exit 0
fi
- #Check regex for rsync temporary file -> ignore
- if [[ ! "${file_name,,}" =~ ^[^\.].*\.(fits|fit|fts).*$ ]]; then
- echo "VALID IGNORE: invalid regular expression"
- exit 0
- fi
+ #Check regex for rsync temporary file -> ignore
+ if [[ ! "${file_name,,}" =~ ^[^\.].*\.(fits|fit|fts).*$ ]]; then
+ echo "VALID IGNORE: invalid regular expression"
+ exit 0
+ fi
+
+ echo "VALID OK"
+ exit 0
- echo "VALID OK"
- exit 0
+# ===========================================================================
elif [ "$1" == "VERIFY" ]; then
- #: Section : VERIFY
- #: Parameter : file path
- #: Response : VERIFY OK
- #: : VERIFY WAIT
- #: : VERIFY FATAL
- #: Description : Check file compliance to fits format
+# ---------------------------------------------------------------------------
+ #: Section : VERIFY
+ #: Parameter : file path
+ #: Response : VERIFY OK
+ #: : VERIFY WAIT
+ #: : VERIFY FATAL
+ #: Description : Check file compliance to fits format
+# ===========================================================================
- file=$2
- FATAL_ERROR="Fatal"
- EOF_ERROR="End-of-file"
+ file=$2
+
+ FATAL_ERROR="Fatal"
+ EOF_ERROR="End-of-file"
#Change file and permission before processing
/usr/bin/sudo -n /bin/chmod go+r $file
- #if fits verify tools exists -> fatal
- if [ ! -x $VERIFY_TOOL ]; then
- echo "VERIFY FATAL : verify tools not exists"
- exit 0
- fi
-
- #if fits file not exists -> fatal
- if [ ! -f $file ]; then
- echo "VERIFY FATAL : file not exists"
- exit 0
- fi
-
- #Check with fits verify
- res=$($VERIFY_TOOL $file 2>&1)
-
- #if fitsverify return fatal error -> wait
- fatal=$(echo $res | grep "$FATAL_ERROR" | wc | awk '{print $1}')
- if [ "$fatal" -ge "1" ]; then
- echo "VERIFY FATAL"
- exit 0
- fi
-
- #if fitsverify return end of file -> wait
- eof=$(echo $res | grep "$EOF_ERROR" | wc | awk '{print $1}')
- if [ "$eof" -ge "1" ]; then
- echo "VERIFY WAIT"
- exit 0
- fi
-
- #else -> ok
- echo "VERIFY OK"
- exit 0
+ #if fits verify tools exists -> fatal
+ if [ ! -x $VERIFY_TOOL ]; then
+ echo "VERIFY FATAL : verify tools not exists"
+ exit 0
+ fi
+
+ #if fits file not exists -> fatal
+ if [ ! -f $file ]; then
+ echo "VERIFY FATAL : file not exists"
+ exit 0
+ fi
+
+ #Check with fits verify
+ res=$($VERIFY_TOOL $file 2>&1)
+
+ #if fitsverify return fatal error -> wait
+ fatal=$(echo $res | grep "$FATAL_ERROR" | wc | awk '{print $1}')
+ if [ "$fatal" -ge "1" ]; then
+ echo "VERIFY FATAL"
+ exit 0
+ fi
+
+ #if fitsverify return end of file -> wait
+ eof=$(echo $res | grep "$EOF_ERROR" | wc | awk '{print $1}')
+ if [ "$eof" -ge "1" ]; then
+ echo "VERIFY WAIT"
+ exit 0
+ fi
+
+ #else -> ok
+ echo "VERIFY OK"
+ exit 0
+
+# ===========================================================================
elif [ "$1" == "PREPROCESS" ]; then
- #: Section : PREPROCESS
- #: Parameter : file path
- #: : ingestion result [OK, WAIT, FATAL]
- #: Response : PREPROCESS OK
- #: : PREPROCESS FATAL
- #: Description : Apply preprocessing before ingestion
-
- file=$2
- file_name=${file##*/}
-
- verified=$3
-
- #Check verified parameter value
- if [ "$verified" != "OK" -a "$verified" != "WAIT" -a "$verified" != "FATAL" ]; then
- echo "PREPROCESS FATAL"
- exit 0
- fi
-
- #Pre processing for verified OK files
- if [ "$verified" == "OK" ]; then
-
- #Change file ownership
- /usr/bin/sudo -n /bin/chown controls:controls $file
- #Change file and permission before processing
- /usr/bin/sudo -n /bin/chmod u+rw $file
-
- #Check regular expression for luci files
- if [[ "${file_name,,}" =~ ^.*luci.*\.(fits|fit|fts).*$ ]]; then
-
- #if listhead tools exists -> fatal
- if [ ! -x $LISTHEAD_TOOL ]; then
- echo "PREPROCESS FATAL : listhead tools not exists"
- exit 0
- fi
-
- #if modhead tools exists -> fatal
- if [ ! -x $MODHEAD_TOOL ]; then
- echo "PREPROCESS FATAL : listhead tools not exists"
- exit 0
- fi
-
- #if fits file not exists -> fatal
- if [ ! -f $file ]; then
- echo "PREPROCESS FATAL : file not exists"
- exit 0
- fi
-
- #Change GRATWLEN key from 'not used' to 0.0
- gratwlen=`$LISTHEAD_TOOL $file 2>&1 | grep -i GRATWLEN`
- if [[ $gratwlen =~ "'not used'/" ]]; then
- $MODHEAD_TOOL $file GRATWLEN 0.0 &>/dev/null
- fi
-
- #Change GRATORDE key from 'not used' to 0.0
- gratorde=`$LISTHEAD_TOOL $file 2>&1 | grep -i GRATORDE`
- if [[ $gratorde =~ "'not used'/" ]]; then
- $MODHEAD_TOOL $file GRATORDE 0.0 &>/dev/null
- fi
- fi #luci files
-
- #Check regular expression for irt files
- if [[ "${file_name,,}" =~ ^.*irt.*\.(fits|fit|fts).*$ ]]; then
-
- #if listhead tools exists -> fatal
- if [ ! -x $LISTHEAD_TOOL ]; then
- echo "PREPROCESS FATAL : listhead tools not exists"
- exit 0
- fi
-
- #if modhead tools exists -> fatal
- if [ ! -x $MODHEAD_TOOL ]; then
- echo "PREPROCESS FATAL : listhead tools not exists"
- exit 0
- fi
-
- #if fits file not exists -> fatal
- if [ ! -f $file ]; then
- echo "PREPROCESS FATAL : file not exists"
- exit 0
- fi
-
- #Search for DATE_OBS keyword
- res=`$LISTHEAD_TOOL $file 2>&1 | grep DATE_OBS | wc | awk '{print $1}'`
- if [ $res -eq 0 ]; then
-
- date=`$LISTHEAD_TOOL $file 2>&1 | grep DATE-OBS`
- time=`$LISTHEAD_TOOL $file 2>&1 | grep TIME-OBS`
-
- if [ ! -z "$date" -a ! -z "$time" ]; then
- date_obs=${date:11:10}T${time:11:12}
- $MODHEAD_TOOL $file DATE_OBS "'$date_obs'" &>/dev/null
- fi
- fi
- fi #irt files
-
- fi #verified ok files
-
- echo "PREPROCESS OK"
- exit 0
+# ---------------------------------------------------------------------------
+
+ #: Section : PREPROCESS
+ #: Parameter : file path
+ #: : ingestion result [OK, WAIT, FATAL]
+ #: Response : PREPROCESS OK
+ #: : PREPROCESS FATAL
+ #: Description : Apply preprocessing before ingestion
+
+
+# ===========================================================================
+
+ file=$2
+ file_name=${file##*/}
+
+ verified=$3
+
+
+
+# ---------------------------------------------------------------------------
+
+
+
+ #Check verified parameter value
+
+ if [ "$verified" != "OK" -a "$verified" != "WAIT" -a "$verified" != "FATAL" ]; then
+ echo "PREPROCESS FATAL"
+ exit 0
+ fi
+
+
+# ---------------------------------------------------------------------------
+
+ #Pre processing for verified OK files
+
+ if [ "$verified" == "OK" ]; then
+
+ #Change file ownership
+ /usr/bin/sudo -n /bin/chown controls:controls $file
+
+ #Change file and permission before processing
+ /usr/bin/sudo -n /bin/chmod u+rw $file
+
+# ---------------------------------------------------------------------------
+
+ #Check regular expression for luci files
+ if [[ "${file_name,,}" =~ ^.*luci.*\.(fits|fit|fts).*$ ]]; then
+
+ #if listhead tools exists -> fatal
+ if [ ! -x $LISTHEAD_TOOL ]; then
+ echo "PREPROCESS FATAL : listhead tools not exists"
+ exit 0
+ fi
+
+ #if modhead tools exists -> fatal
+ if [ ! -x $MODHEAD_TOOL ]; then
+ echo "PREPROCESS FATAL : listhead tools not exists"
+ exit 0
+ fi
+
+
+ #if fits file not exists -> fatal
+ if [ ! -f $file ]; then
+ echo "PREPROCESS FATAL : file not exists"
+ exit 0
+ fi
+
+ #Change GRATWLEN key from 'not used' to 0.0
+ gratwlen=`$LISTHEAD_TOOL $file 2>&1 | grep -i GRATWLEN`
+ if [[ $gratwlen =~ "'not used'/" ]]; then
+ $MODHEAD_TOOL $file GRATWLEN 0.0 &>/dev/null
+ fi
+
+ #Change GRATORDE key from 'not used' to 0.0
+ gratorde=`$LISTHEAD_TOOL $file 2>&1 | grep -i GRATORDE`
+ if [[ $gratorde =~ "'not used'/" ]]; then
+ $MODHEAD_TOOL $file GRATORDE 0.0 &>/dev/null
+ fi
+
+ fi #luci files
+
+# ---------------------------------------------------------------------------
+
+ #Check regular expression for irt files
+ if [[ "${file_name,,}" =~ ^.*irt.*\.(fits|fit|fts).*$ ]]; then
+
+ #if listhead tools exists -> fatal
+ if [ ! -x $LISTHEAD_TOOL ]; then
+ echo "PREPROCESS FATAL : listhead tools not exists"
+ exit 0
+ fi
+
+ #if modhead tools exists -> fatal
+ if [ ! -x $MODHEAD_TOOL ]; then
+ echo "PREPROCESS FATAL : listhead tools not exists"
+ exit 0
+ fi
+
+
+ #if fits file not exists -> fatal
+ if [ ! -f $file ]; then
+ echo "PREPROCESS FATAL : file not exists"
+ exit 0
+ fi
+
+ #Search for DATE_OBS keyword
+ res=`$LISTHEAD_TOOL $file 2>&1 | grep DATE_OBS | wc | awk '{print $1}'`
+ if [ $res -eq 0 ]; then
+
+ date=`$LISTHEAD_TOOL $file 2>&1 | grep DATE-OBS`
+ time=`$LISTHEAD_TOOL $file 2>&1 | grep TIME-OBS`
+
+ if [ ! -z "$date" -a ! -z "$time" ]; then
+ date_obs=${date:11:10}T${time:11:12}
+ $MODHEAD_TOOL $file DATE_OBS "'$date_obs'" &>/dev/null
+ fi
+ fi
+
+
+
+ fi #irt files
+
+
+# ---------------------------------------------------------------------------
+
+
+ fi #verified ok files
+
+
+# ---------------------------------------------------------------------------
+
+ #: Sub Section : PREPROCESS - Wright CHECKSUM and DATASUM
+ #: Parameter : file path
+ #: Response : PREPROCESS OK
+ #: : PREPROCESS FATAL
+ #: Description : Wrighting CHECKSUM and DATASUM in $file before moving it
+
+
+# ===========================================================================
+
+
+ file=$2
+ file_name=${file##*/}
+
+
+ #if fitscheck tools exists -> fatal
+ if [ ! -x $FITSCHECK_TOOL ]; then
+ echo "PREPROCESS FATAL : fitscheck tools not exists"
+ exit 0
+ fi
+
+
+ #if listhead tools exists -> fatal
+ if [ ! -x $LISTHEAD_TOOL ]; then
+ echo "PREPROCESS FATAL : listhead tools not exists"
+ exit 0
+ fi
+
+
+ #if fits file not exists -> fatal
+ if [ ! -f $file ]; then
+ echo "PREPROCESS FATAL : file not exists"
+ exit 0
+ fi
+
+
+ new_file=$($FITSCHECK_TOOL -vf $file )
+ datasum=$($LISTHEAD_TOOL $file| grep DATASUM)
+ datasum_value=$($LISTHEAD_TOOL $file| grep DATASUM|awk '{print $3}')
+ checkdatasum=$(echo $datasum | grep DATASUM | wc | awk '{print $1}')
+
+
+
+ if [ "$checkdatasum" -lt "1" ]; then
+ echo "DATASUM is NOT recorded in $file"
+ exit 0 ; else
+ echo "DATASUM is recorded: $datasum_value"
+ fi
+
+# ---------------------------------------------------------------------------
+
+ echo "PREPROCESS OK"
+ exit 0
+
+
+# ===========================================================================
elif [ "$1" == "POSTPROCESS" ]; then
- #: Section : POSTPROCESS
- #: Parameter : file path
- #: : ingestion result [OK, WAIT, FATAL]
- #: Response : POSTPROCESS OK
- #: : POSTPROCESS FATAL
- #: Description : Apply postprocessing after ingestion
+# ---------------------------------------------------------------------------
+
+ #: Section : POSTPROCESS
+ #: Parameter : file path
+ #: : ingestion result [OK, WAIT, FATAL]
+ #: Response : POSTPROCESS OK
+ #: : POSTPROCESS FATAL
+ #: Description : Apply postprocessing after ingestion
+
+# ===========================================================================
+
+ file=$2
+ file_name=${file##*/}
- file=$2
- file_name=${file##*/}
+ verified=$3
- verified=$3
+ #Check verified parameter value
- #Check verified parameter value
- if [ "$verified" != "OK" -a "$verified" != "WAIT" -a "$verified" != "FATAL" ]; then
- echo "POSTPROCESS FATAL"
- exit 0
- fi
+ if [ "$verified" != "OK" -a "$verified" != "WAIT" -a "$verified" != "FATAL" ]; then
+ echo "POSTPROCESS FATAL"
+ exit 0
+ fi
- #Post process verified WAIT files
- if [ "$verified" == "WAIT" ]; then
- echo "Warning file: $file_name" | mutt -s "Pre process log" -- demarco@oats.inaf.it
- fi
+# ---------------------------------------------------------------------------
- #Post process verified FATAL files
- if [ "$verified" == "FATAL" ]; then
- echo "Fatal file: $file_name" | mutt -s "Pre process log" -- demarco@oats.inaf.it
- fi
+ #Post process verified WAIT files
+ if [ "$verified" == "WAIT" ]; then
+ echo "Warning file: $file_name" | mutt -s "Pre process log" -- demarco@oats.inaf.it
+ fi
- echo "POSTPROCESS OK"
- exit 0
+# ---------------------------------------------------------------------------
+
+ #Post process verified FATAL files
+ if [ "$verified" == "FATAL" ]; then
+ echo "Fatal file: $file_name" | mutt -s "Pre process log" -- demarco@oats.inaf.it
+ fi
+
+
+ echo "POSTPROCESS OK"
+ exit 0
+
+# ===========================================================================
else
- #: Section : DEFAULT
- #: Parameter : none
- #: Response : UNKNOWN
+# ---------------------------------------------------------------------------
+
+ #: Section : DEFAULT
+ #: Parameter : none
+ #: Response : UNKNOWN
+
+# ===========================================================================
+
+ echo "UNKNOWN"
+ exit 0
- echo "UNKNOWN"
- exit 0
fi
--
GitLab