From 1a6fb374154434df7b15434ee12655df1f702df2 Mon Sep 17 00:00:00 2001
From: Cristiano Urban <cristiano.urban@inaf.it>
Date: Fri, 8 Apr 2022 12:08:42 +0200
Subject: [PATCH] Minor changes + fixed bug in 'setAsyncTrans()' method: ingore
 local nodes.

Signed-off-by: Cristiano Urban <cristiano.urban@inaf.it>
---
 transfer_service/db_connector.py     | 12 +++++-
 transfer_service/retrieve_cleaner.py | 58 ++++++++++++++--------------
 2 files changed, 40 insertions(+), 30 deletions(-)

diff --git a/transfer_service/db_connector.py b/transfer_service/db_connector.py
index 59332b2..fb7ded0 100644
--- a/transfer_service/db_connector.py
+++ b/transfer_service/db_connector.py
@@ -1084,14 +1084,22 @@ class DbConnector(object):
             self.connPool.putconn(conn, close = False)
 
     def setAsyncTrans(self, nodeVOSPath, value):
-        """Sets the 'async_trans' flag for a VOSpace node."""
+        """
+        Sets the 'async_trans' flag recursively for a VOSpace node.
+        Local nodes are not considered.
+        """
         try:
             conn = self.getConnection()
             cursor = conn.cursor(cursor_factory = RealDictCursor)
             cursor.execute("""
                 UPDATE node c SET async_trans = %s
                 FROM node n
-                WHERE c.path <@ n.path AND n.node_id = id_from_vos_path(%s);
+                WHERE c.path <@ n.path
+                AND n.node_id = id_from_vos_path(%s)
+                AND c.location_id NOT IN
+                (SELECT location_id
+                 FROM location
+                 WHERE storage_src_id = storage_dest_id);
                 """,
                 (value, nodeVOSPath,))
             conn.commit()
diff --git a/transfer_service/retrieve_cleaner.py b/transfer_service/retrieve_cleaner.py
index 7e0e856..179c794 100644
--- a/transfer_service/retrieve_cleaner.py
+++ b/transfer_service/retrieve_cleaner.py
@@ -10,6 +10,7 @@ import json
 import logging
 import os
 import shutil
+import time
 
 from config import Config
 from db_connector import DbConnector
@@ -86,40 +87,41 @@ class RetrieveCleaner(TaskExecutor):
                 return False
             numNodes = len(self.nodeList)
             self.logger.info(f"Number of VOSpace nodes involved: {numNodes}")
-            while numNodes > 0:                    
-                i = 0
-                while i < numNodes:
-                    vospacePath = self.nodeList[i]
-                    destPath = self.destPathList[i]
-                    try:
-                        busy = self.dbConn.nodeIsBusy(vospacePath)
-                    except Exception:
-                        self.logger.exception(f"FATAL: unable to check the 'busy' flag value for the VOSpace node '{vospacePath}'.")
-                        return False
+            while numNodes > 0:
+                time.sleep(0.2)
+                vospacePath = self.nodeList[numNodes - 1]
+                destPath = self.destPathList[numNodes - 1]
+                try:
+                    busy = self.dbConn.nodeIsBusy(vospacePath)
+                except Exception:
+                    self.logger.exception(f"FATAL: unable to check the 'busy' flag value for the VOSpace node '{vospacePath}'.")
+                    return False
+                else:
                     if not busy:
                         try:
                             self.dbConn.setAsyncTrans(vospacePath, True)
                         except Exception:
                             self.logger.exception(f"FATAL: unable to update the 'async_trans' flag for the VOSpace node '{vospacePath}'.")
                             return False
-                        try:
-                            if os.path.isfile(destPath):
-                                os.remove(destPath)
-                            else:
-                                shutil.rmtree(destPath)
-                        except FileNotFoundError:
-                            self.logger.exception(f"Cannot find '{destPath}', skip...")
-                        # check for empty dirs and remove them
-                        basePath = self.storageRetrievePath.replace("{username}", self.username)
-                        for root, dirs, files in os.walk(basePath, topdown = False):
-                            for dir in dirs:
-                                dirPath = os.path.abspath(root) + '/' + dir
-                                if not os.listdir(dirPath):
-                                    os.rmdir(dirPath)
-                        self.nodeList.pop(i)
-                        self.destPathList.pop(i)
-                        numNodes -= 1
-                        i = 0
+                        else:
+                            try:
+                                if os.path.isfile(destPath):
+                                    os.remove(destPath)
+                                else:
+                                    shutil.rmtree(destPath)
+                            except FileNotFoundError:
+                                self.logger.exception(f"Cannot find '{destPath}', skip...")
+
+                            # check for empty dirs and remove them
+                            basePath = self.storageRetrievePath.replace("{username}", self.username)
+                            for root, dirs, files in os.walk(basePath, topdown = False):
+                                for dir in dirs:
+                                    dirPath = os.path.abspath(root) + '/' + dir
+                                    if not os.listdir(dirPath):
+                                        os.rmdir(dirPath)
+                            self.nodeList.pop(numNodes - 1)
+                            self.destPathList.pop(numNodes - 1)
+                            numNodes -= 1
         except Exception:
             self.logger.exception("FATAL: something went wrong while cleaning the expired data.")
             return False
-- 
GitLab