diff --git a/base/Dockerfile b/base/Dockerfile
index 9c30986538f2149a52f0e257607b5ee8fcfc1dd6..48a3736f5363d7041489fc8c4b88604a5a198ba2 100644
--- a/base/Dockerfile
+++ b/base/Dockerfile
@@ -7,6 +7,16 @@
# Use CentOS 7 as base image
FROM centos:7
+# Copy the repository file to the container
+COPY vault.repo /root/vault.repo
+
+# Update the system and install basic utilities
+RUN yum-config-manager --disable base && \
+ yum-config-manager --disable updates && \
+ yum-config-manager --disable extras && \
+ cat /root/vault.repo >> /etc/yum.repos.d/Centos-Vault.repo && \
+ yum clean all && yum makecache
+
# Install epel repo
RUN yum update -y && yum -y install epel-release
diff --git a/base/vault.repo b/base/vault.repo
new file mode 100644
index 0000000000000000000000000000000000000000..5c1d6910bd6eaf195c6010757dff56cac06bfc79
--- /dev/null
+++ b/base/vault.repo
@@ -0,0 +1,15 @@
+[Vault-base]
+name=Vault - CentOS-$releasever Base
+baseurl=http://vault.centos.org/centos/$releasever/os/$basearch/
+gpgcheck=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-$releasever
+[Vault-updates]
+name=Vault - CentOS-$releasever Updates
+baseurl=http://vault.centos.org/centos/$releasever/updates/$basearch/
+gpgcheck=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-$releasever
+[Vault-extras]
+name=Vault - CentOS-$releasever Extras
+baseurl=http://vault.centos.org/centos/$releasever/extras/$basearch/
+gpgcheck=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-$releasever
diff --git a/client/vos_storage b/client/vos_storage
index 5f1e31e070176484c6b15f3b416d2f3e97f3b99c..9d43221d9ef649daa02b3fa8967e565ff1120d88 100644
--- a/client/vos_storage
+++ b/client/vos_storage
@@ -28,8 +28,8 @@ class VOSStorage(RedisRPCClient):
storageType = None
storageBasePath = None
storageHostname = None
- tapePool = None
- tapePoolList = []
+ tapeHSMFilesystem = None
+ tapeHSMFilesystemList = []
while storageType not in ("cold", "hot"):
try:
storageType = input("\nStorage type ['cold' or 'hot']: ")
@@ -40,17 +40,17 @@ class VOSStorage(RedisRPCClient):
except KeyboardInterrupt:
sys.exit("\nCTRL+C detected. Exiting...")
if storageType == "cold":
- storageRequest = { "requestType": "TAPE_POOL_LST" }
+ storageRequest = { "requestType": "TAPE_HSM_FS_LST" }
storageResponse = self.call(storageRequest)
if "responseType" not in storageResponse:
sys.exit("FATAL: Malformed response, storage acknowledge expected.\n")
- elif storageResponse["responseType"] == "TAPE_POOL_LST_DONE":
- tapePoolList = storageResponse["tapePoolList"]
- while tapePool not in tapePoolList:
- print("\nSelect one of the available tape pools:")
- print("\n" + tabulate([ [t] for t in tapePoolList ], headers = ["tape_pool"], tablefmt = "pretty") + "\n")
+ elif storageResponse["responseType"] == "TAPE_HSM_FS_LST_DONE":
+ tapeHSMFilesystemList = storageResponse["tapeHSMFilesystemList"]
+ while tapeHSMFilesystem not in tapeHSMFilesystemList:
+ print("\nSelect one of the available tape HSM filesystems:")
+ print("\n" + tabulate([ [t] for t in tapeHSMFilesystemList ], headers = ["tape_hsm_fs"], tablefmt = "pretty") + "\n")
try:
- tapePool = input("Please, insert a tape pool: ")
+ tapeHSMFilesystem = input("Please, insert a tape HSM filesystem: ")
except ValueError:
print("Input type is not valid!")
except EOFError:
@@ -81,7 +81,7 @@ class VOSStorage(RedisRPCClient):
"storageType": storageType,
"basePath": storageBasePath,
"hostname": storageHostname,
- "tapePool": tapePool }
+ "tapeHSMFilesystem": tapeHSMFilesystem }
storageResponse = self.call(storageRequest)
if "responseType" not in storageResponse:
@@ -202,8 +202,8 @@ DESCRIPTION
Adding 'hot' or 'cold' storage points requires a base path
to be specified (e.g. '/mnt/my_storage/users').
- Adding a 'cold' storage point requires selecting a tape pool
- (a tape pool list is provided).
+ Adding a 'cold' storage point requires selecting a tape HSM
+ filesystem (a tape HSM filesystem list is provided).
All storage points require a valid hostname or IP address.
""")
diff --git a/transfer_service/config/vos_ts.conf.sample b/transfer_service/config/vos_ts.conf.sample
index 6f9adf43c522a23c79a4268c0b028a2e528c83ae..14e6c4e432d5ab4132c963c9d39cfe5c57f0e615 100644
--- a/transfer_service/config/vos_ts.conf.sample
+++ b/transfer_service/config/vos_ts.conf.sample
@@ -30,8 +30,8 @@ port = 6379
; db index representing the db that stores the scheduling queues, default is 0
db_sched = 0
-# Spectrum Archive
-[spectrum_archive]
+# IBM Spectrum Protect
+[spectrum_protect]
; hostname or IP address of the tape library frontend
host = <tape_frontend_hostname>
; SSH port
diff --git a/transfer_service/db_connector.py b/transfer_service/db_connector.py
index 179dfcfd25ff3a62497a3adb8387156632bc7b0e..15a82bc7d938d0232fecf5bf3a257a3140e86c0f 100644
--- a/transfer_service/db_connector.py
+++ b/transfer_service/db_connector.py
@@ -171,7 +171,7 @@ class DbConnector(object):
conn = self.getConnection()
cursor = conn.cursor(cursor_factory = RealDictCursor)
cursor.execute("""
- SELECT storage_type, base_path, user_name, tstamp_wrapper_dir, '/' || fs_path AS os_path, content_length, tape_pool
+ SELECT storage_type, base_path, user_name, tstamp_wrapper_dir, '/' || fs_path AS os_path, content_length, tape_hsm_fs
FROM node n
JOIN location l ON n.location_id = l.location_id
JOIN storage s ON s.storage_id = l.storage_src_id
@@ -192,7 +192,7 @@ class DbConnector(object):
tstampWrappedDir = result[0]["tstamp_wrapper_dir"]
osPath = result[0]["os_path"]
contentLength = result[0]["content_length"]
- tapePool = result[0]["tape_pool"]
+ tapeHSMFilesystem = result[0]["tape_hsm_fs"]
if tstampWrappedDir is None:
baseSrcPath = basePath + "/" + userName
else:
@@ -205,7 +205,7 @@ class DbConnector(object):
"username": userName,
"osPath": osPath,
"contentLength": contentLength,
- "tapePool": tapePool
+ "tapeHSMFilesystem": tapeHSMFilesystem
}
return fileInfo
finally:
@@ -824,12 +824,12 @@ class DbConnector(object):
finally:
self.connPool.putconn(conn, close = False)
- def getTapePool(self, storageId):
- """Returns the tape pool for a given storage id, if the storage is cold. Otherwise it returns 'False'."""
+ def getTapeHSMFilesystem(self, storageId):
+ """Returns the tape HSM filesystem for a given storage id, if the storage is cold. Otherwise it returns 'False'."""
try:
conn = self.getConnection()
cursor = conn.cursor(cursor_factory = RealDictCursor)
- cursor.execute("SELECT tape_pool FROM storage WHERE storage_id = %s;", (storageId,))
+ cursor.execute("SELECT tape_hsm_fs FROM storage WHERE storage_id = %s;", (storageId,))
result = cursor.fetchall()
cursor.close()
except Exception:
@@ -838,7 +838,7 @@ class DbConnector(object):
raise
else:
if result:
- return result[0]["tape_pool"]
+ return result[0]["tape_hsm_fs"]
else:
return False
finally:
@@ -1250,7 +1250,7 @@ class DbConnector(object):
##### Storage #####
- def insertStorage(self, storageType, storageBasePath, storageHostname, vospaceUserBasePath, tapePool = None):
+ def insertStorage(self, storageType, storageBasePath, storageHostname, vospaceUserBasePath, tapeHSMFilesystem = None):
"""Inserts a storage point."""
if not self.getStorageId(storageBasePath):
try:
@@ -1260,14 +1260,14 @@ class DbConnector(object):
INSERT INTO storage(storage_type,
base_path,
hostname,
- tape_pool)
+ tape_hsm_fs)
VALUES (%s, %s, %s, %s)
RETURNING storage_id;
""",
(storageType,
storageBasePath,
storageHostname,
- tapePool,))
+ tapeHSMFilesystem,))
storageSrcId = cursor.fetchall()[0]["storage_id"]
except Exception:
if not conn.closed:
diff --git a/transfer_service/doi_executor.py b/transfer_service/doi_executor.py
index 14a999461e365117d302479f1170374765415a96..8d3ea79859c71ca5ef007c2a7e8843101a725d1c 100644
--- a/transfer_service/doi_executor.py
+++ b/transfer_service/doi_executor.py
@@ -71,7 +71,7 @@ class DOIExecutor(TaskExecutor):
1,
1,
self.logger)
- params = config.loadSection("spectrum_archive")
+ params = config.loadSection("spectrum_protect")
#self.tapePool = params["tape_pool"]
self.tapeClient = TapeClient(params["host"],
params.getint("port"),
diff --git a/transfer_service/import_executor.py b/transfer_service/import_executor.py
index 6deb10c47593756bf0dcc4b010eb61e7d1146458..ed9c43008b02a29395aa849a4c79b7bcd4f5da7a 100644
--- a/transfer_service/import_executor.py
+++ b/transfer_service/import_executor.py
@@ -50,7 +50,7 @@ class ImportExecutor(TaskExecutor):
1,
1,
self.logger)
- params = config.loadSection("spectrum_archive")
+ params = config.loadSection("spectrum_protect")
self.tapeClient = TapeClient(params["host"],
params.getint("port"),
params["user"],
diff --git a/transfer_service/retrieve_executor.py b/transfer_service/retrieve_executor.py
index 0b5378a42fd9eef374280e034df51ac9bd19c2b7..0eac4ea6608d5c33cf729574c87bf4bcec488429 100644
--- a/transfer_service/retrieve_executor.py
+++ b/transfer_service/retrieve_executor.py
@@ -75,13 +75,13 @@ class RetrieveExecutor(TaskExecutor):
1,
1,
self.logger)
- params = config.loadSection("spectrum_archive")
+ params = config.loadSection("spectrum_protect")
self.tapeClient = TapeClient(params["host"],
params.getint("port"),
params["user"],
params["pkey_file_path"],
self.logger)
- self.tapePool = None
+ self.tapeHSMFilesystem = None
self.storageType = None
self.jobObj = None
self.jobId = None
@@ -125,7 +125,7 @@ class RetrieveExecutor(TaskExecutor):
#self.storageType = self.dbConn.getOSPath(self.nodeList[0])["storageType"]
fileInfo = self.dbConn.getOSPath(self.nodeList[0])
self.storageType = fileInfo["storageType"]
- self.tapePool = fileInfo["tapePool"]
+ self.tapeHSMFilesystem = fileInfo["tapeHSMFilesystem"]
except Exception:
self.logger.exception("FATAL: unable to obtain the storage type.")
return False
@@ -321,7 +321,7 @@ class RetrieveExecutor(TaskExecutor):
# is 'cold'
if self.storageType == "cold":
self.tapeClient.connect()
- self.tapeClient.migrate([ f["fullPath"] for f in blockFileList if f["fileSize"] > 0 ], self.tapePool, self.jobId)
+ self.tapeClient.migrate([ f["fullPath"] for f in blockFileList if f["fileSize"] > 0 ], self.tapeHSMFilesystem, self.jobId)
self.tapeClient.disconnect()
blockFileList.clear()
diff --git a/transfer_service/storage_rpc_server.py b/transfer_service/storage_rpc_server.py
index d04b9eedadb7c73ed294607da9afb7df65961902..6ee804a90c68cefbd1c063225a8febdc5016429e 100644
--- a/transfer_service/storage_rpc_server.py
+++ b/transfer_service/storage_rpc_server.py
@@ -40,7 +40,7 @@ class StorageRPCServer(RedisRPCServer):
1,
2,
self.logger)
- params = config.loadSection("spectrum_archive")
+ params = config.loadSection("spectrum_protect")
self.tapeClient = TapeClient(params["host"],
params.getint("port"),
params["user"],
@@ -135,21 +135,20 @@ class StorageRPCServer(RedisRPCServer):
else:
response = { "responseType": "STORAGE_LST_DONE",
"storageList": result }
- elif requestBody["requestType"] == "TAPE_POOL_LST":
+ elif requestBody["requestType"] == "TAPE_HSM_FS_LST":
try:
self.tapeClient.connect()
- tapePools = self.tapeClient.getPoolList()
+ tapeHSMFilesystemList = self.tapeClient.getHSMFilesystemList()
self.tapeClient.disconnect()
- tapePoolList = [ p.getName() for p in tapePools ]
except Exception:
- errorMsg = "Unable to get tape pool list."
+ errorMsg = "Unable to get tape HSM filesystem list."
self.logger.exception(errorMsg)
response = { "responseType": "ERROR",
"errorCode": 6,
"errorMsg": errorMsg }
else:
- response = { "responseType": "TAPE_POOL_LST_DONE",
- "tapePoolList": tapePoolList }
+ response = { "responseType": "TAPE_HSM_FS_LST_DONE",
+ "tapeHSMFilesystemList": tapeHSMFilesystemList }
else:
errorMsg = "Unkown request type."
self.logger.error(errorMsg)
diff --git a/transfer_service/store_executor.py b/transfer_service/store_executor.py
index 1767e24ebc557c6345b089a3cf2c0a3d9cf1f322..4f7aac290329ebb29679799a88927e83f0fd3015 100644
--- a/transfer_service/store_executor.py
+++ b/transfer_service/store_executor.py
@@ -24,7 +24,7 @@ from task_executor import TaskExecutor
class StoreExecutor(TaskExecutor):
-
+
# We use 10 GB of tolerance when we calculate
# the free space in our storage point
TOL = 10 * (2**30)
@@ -32,7 +32,7 @@ class StoreExecutor(TaskExecutor):
def __init__(self):
self.type = "store_executor"
self.systemUtils = SystemUtils()
- config = Config("/etc/vos_ts/vos_ts.conf")
+ config = Config("/etc/vos_ts/vos_ts.conf")
params = config.loadSection("transfer_node")
self.storageStorePath = params["store_path"]
params = config.loadSection("transfer")
@@ -60,7 +60,7 @@ class StoreExecutor(TaskExecutor):
1,
1,
self.logger)
- params = config.loadSection("spectrum_archive")
+ params = config.loadSection("spectrum_protect")
self.tapeClient = TapeClient(params["host"],
params.getint("port"),
params["user"],
@@ -104,18 +104,15 @@ class StoreExecutor(TaskExecutor):
storageFreeSpace = free - self.TOL
self.logger.debug(f"storageFreeSpace (hot): {storageFreeSpace} B")
else:
- self.tapeClient.connect()
- tapePoolList = self.tapeClient.getPoolList()
- self.tapeClient.disconnect()
try:
- self.tapePool = self.dbConn.getTapePool(self.storageId)
+ self.tapeHSMFilesystem = self.dbConn.getTapeHSMFilesystem(self.storageId)
except Exception:
- self.logger.exception("FATAL: unable to obtain the tape pool.")
+ self.logger.exception("FATAL: unable to obtain the tape HSM filesystem.")
return False
else:
- for el in tapePoolList:
- if el.getName() == self.tapePool:
- storageFreeSpace = el.getFreeSpace() - self.TOL
+ self.tapeClient.connect()
+ storageFreeSpace = self.tapeClient.getHSMFilesystemFreeSpace(self.tapeHSMFilesystem) - self.TOL
+ self.tapeClient.disconnect()
self.logger.debug(f"storageFreeSpace (cold): {storageFreeSpace} B")
if storageFreeSpace < self.dataSize:
self.logger.error("FATAL: space available on the storage point is not enough.")
diff --git a/transfer_service/tape_client.py b/transfer_service/tape_client.py
index 2c46e75cb29e2cb60749a4dd2357be4f032a3c2c..37f0dec7043f980004087de7a50b1010413e4370 100644
--- a/transfer_service/tape_client.py
+++ b/transfer_service/tape_client.py
@@ -16,13 +16,19 @@ from config import Config
from exceptions import ScpInvalidFileException
from exceptions import TapeClientException
from redis_log_handler import RedisLogHandler
-from tape_pool import TapePool
-from tape_task import TapeTask
+from system_utils import SystemUtils
class TapeClient(object):
- # 'eeadm' command location on the tape library frontend
- EEADM = "/opt/ibm/ltfsee/bin/eeadm"
+ # 'dsmdf' command location on the tape library frontend
+ DSMDF = "/usr/bin/dsmdf"
+
+ # 'dsmrecall' command location on the tape library frontend
+ DSMRECALL = "/usr/bin/dsmrecall"
+
+ # 'dsmmigrate' command location on the tape library frontend
+ DSMMIGRATE = "/usr/bin/dsmmigrate"
+
# destination for the files containing the lists of files to recall or migrate
VOSPACE_WD = "/tmp/vospace"
@@ -35,8 +41,8 @@ class TapeClient(object):
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.keyFile = keyFile
self.scp = None
- self.taskList = []
- self.poolList = []
+ self.systemUtils = SystemUtils()
+ self.HSMFilesystemList = []
def connect(self):
"""Connects to the tape library frontend."""
@@ -51,51 +57,28 @@ class TapeClient(object):
self.logger.exception("Unable to establish SSH connection with tape library frontend.")
raise
- def getPoolList(self):
- """Returns a list of 'TapePool' objects."""
- cmd = f"{self.EEADM} pool list --json"
+ def copy(self, srcPath, destPath):
+ """Copies files/dirs recursively by passing their absolute paths."""
try:
- stdin, stdout, stderr = self.client.exec_command(cmd)
+ self.scp = scp.SCPClient(self.client.get_transport())
except Exception:
- self.logger.exception(f"Unable to execute command: '{cmd}'")
+ self.logger.error("Unable to get transport from SSH client.")
raise
else:
- exitCode = stdout.channel.recv_exit_status()
- if not exitCode:
- result = json.loads(stdout.readlines()[0].rstrip('\n'))
- for el in result["payload"]:
- pool = TapePool()
- pool.id = el["id"]
- pool.name = el["name"]
- pool.mediaRestriction = el["media_restriction"]
- pool.capacity = el["capacity"]
- pool.usedSpace = el["used_space"]
- pool.freeSpace = el["free_space"]
- pool.reclaimableSpace = el["reclaimable_space"]
- pool.activeSpace = el["active_space"]
- pool.nonAppendableSpace = el["non_appendable_space"]
- pool.numOfTapes = el["num_of_tapes"]
- pool.formatClass = el["format_class"]
- pool.libraryName = el["library_name"]
- pool.libraryId = el["library_id"]
- pool.nodeGroupName = el["nodegroup_name"]
- pool.deviceType = el["device_type"]
- pool.worm = el["worm"]
- pool.fillPolicy = el["fill_policy"]
- pool.owner = el["owner"]
- pool.mountLimit = el["mount_limit"]
- pool.lowSpaceWarningEnable = el["low_space_warning_enable"]
- pool.lowSpaceWarningThreshold = el["low_space_warning_threshold"]
- pool.noSpaceWarningEnable = el["no_space_warning_enable"]
- pool.mode = el["mode"]
- self.poolList.append(pool)
- return self.poolList.copy()
+ self.logger.debug(f"Copying {srcPath} in {destPath}")
+ if os.path.isdir(srcPath):
+ self.scp.put(srcPath, recursive = True, remote_path = destPath)
+ elif os.path.isfile(srcPath):
+ self.scp.put(srcPath, destPath)
else:
- raise TapeClientException(cmd, exitCode, stderr)
+ self.logger.error("FATAL: invalid file/dir.")
+ raise ScpInvalidFileException
+ finally:
+ self.scp.close()
- def getTaskList(self):
- """Returns the whole task list."""
- cmd = f"{self.EEADM} task list --json"
+ def getHSMFilesystemList(self):
+ """Returns a list containing all the available HSM filesystems."""
+ cmd = f"{self.DSMDF} -detail | grep \"HSM Filesystem\" | awk '{{ print $3 }}'"
try:
stdin, stdout, stderr = self.client.exec_command(cmd)
except Exception:
@@ -104,55 +87,37 @@ class TapeClient(object):
else:
exitCode = stdout.channel.recv_exit_status()
if not exitCode:
- result = json.loads(stdout.readlines()[0].rstrip('\n'))
- for el in result["payload"]:
- task = TapeTask()
- task.inUseTapes = el["inuse_tapes"]
- task.inUsePools = el["inuse_pools"]
- task.inUseNodeGroups = el["inuse_node_groups"]
- task.inUseDrives = el["inuse_drives"]
- task.cmdParam = el["cmd_param"]
- task.result = el["result"]
- task.status = el["status"]
- task.completedTime = el["completed_time"]
- task.startedTime = el["started_time"]
- task.createdTime = el["created_time"]
- task.setInUseLibs = el["inuse_libs"]
- task.type = el["type"]
- task.taskId = el["task_id"]
- task.id = el["id"]
- self.taskList.append(task)
- return self.taskList.copy()
+ result = stdout.readlines()[0].rstrip('\n')
+ self.HSMFilesystemList = result.splitlines()
+ return self.HSMFilesystemList.copy()
else:
raise TapeClientException(cmd, exitCode, stderr)
- def copy(self, srcPath, destPath):
- """Copies files/dirs recursively by passing their absolute paths."""
+ def getHSMFilesystemFreeSpace(self, HSMFilesystem):
+ "Returns the free space in bytes for a given HSM filesystem."
+ cmd = f"{self.DSMDF} -detail {HSMFilesystem} | grep \"Free Size\" | awk '{{ print $3 }}'"
try:
- self.scp = scp.SCPClient(self.client.get_transport())
+ stdin, stdout, stderr = self.client.exec_command(cmd)
except Exception:
- self.logger.error("Unable to get transport from SSH client.")
+ self.logger.exception(f"Unable to execute command: '{cmd}'")
raise
else:
- self.logger.debug(f"Copying {srcPath} in {destPath}")
- if os.path.isdir(srcPath):
- self.scp.put(srcPath, recursive = True, remote_path = destPath)
- elif os.path.isfile(srcPath):
- self.scp.put(srcPath, destPath)
+ exitCode = stdout.channel.recv_exit_status()
+ if not exitCode:
+ result = stdout.readlines()[0].rstrip('\n') + " KB"
+ freeSpace = self.systemUtils.convertSizeToBytes(result)
+ return freeSpace
else:
- self.logger.error("FATAL: invalid file/dir.")
- raise ScpInvalidFileException
- finally:
- self.scp.close()
+ raise TapeClientException(cmd, exitCode, stderr)
- def migrate(self, fileList, tapePool, jobId):
+ def migrate(self, fileList, tapeHSMFilesystem, jobId):
"""
Migrates to tape all files whose absolute path is
contained in 'fileList'.
- A tape pool and a VOSpace jobId are also required
- as parameters.
+ A HSM filesystem and a VOSpace jobId are also
+ required as parameters.
"""
- self.logger.info(f"Starting MIGRATE operation (tape pool = '{tapePool}')...")
+ self.logger.info(f"Starting MIGRATE operation (tape HSM filesystem = '{tapeHSMFilesystem}')...")
migrateFileList = f"vos_migrate-{jobId}.lst"
try:
fp = open(migrateFileList, "a")
@@ -164,7 +129,7 @@ class TapeClient(object):
fp.close()
self.copy(f"./{migrateFileList}", f"{self.VOSPACE_WD}/{migrateFileList}")
os.remove(f"./{migrateFileList}")
- cmd = f"{self.EEADM} migrate {self.VOSPACE_WD}/{migrateFileList} -p {tapePool} > /dev/null 2>&1"
+ cmd = f"{self.DSMMIGRATE} -filelist={self.VOSPACE_WD}/{migrateFileList} > /dev/null 2>&1"
try:
stdin, stdout, stderr = self.client.exec_command(cmd)
except Exception:
@@ -172,7 +137,7 @@ class TapeClient(object):
raise
else:
exitCode = stdout.channel.recv_exit_status()
- if not exitCode:
+ if exitCode == 0 or exitCode == 4:
self.logger.info("MIGRATE operation COMPLETED.")
else:
self.logger.error("MIGRATE operation FAILED.")
@@ -197,7 +162,7 @@ class TapeClient(object):
fp.close()
self.copy(f"./{recallFileList}", f"{self.VOSPACE_WD}/{recallFileList}")
os.remove(f"./{recallFileList}")
- cmd = f"{self.EEADM} recall {self.VOSPACE_WD}/{recallFileList} > /dev/null 2>&1"
+ cmd = f"{self.DSMRECALL} -filelist={self.VOSPACE_WD}/{recallFileList} > /dev/null 2>&1"
try:
stdin, stdout, stderr = self.client.exec_command(cmd)
except Exception:
@@ -205,7 +170,7 @@ class TapeClient(object):
raise
else:
exitCode = stdout.channel.recv_exit_status()
- if not exitCode:
+ if exitCode == 0 or exitCode == 4:
self.logger.info("RECALL operation COMPLETED.")
else:
self.logger.error("RECALL operation FAILED.")
@@ -218,7 +183,8 @@ class TapeClient(object):
the 'dirName' directory.
"""
self.logger.info("Starting RECALL_CHECKSUM operation...")
- cmd = f"find $(dirname {dirName}) -type f \( -iname \"*-md5sum.txt\" \) | {self.EEADM} recall > /dev/null 2>&1"
+ checksumFileList = "vos_recall_checksum_files-{jobId}.lst"
+ cmd = f"find $(dirname {dirName}) -type f \( -iname \"*-md5sum.txt\" \) > {self.VOSPACE_WD}/{checksumFileList} && {self.DSMRECALL} -filelist={self.VOSPACE_WD}/{checksumFileList} > /dev/null 2>&1"
try:
stdin, stdout, stderr = self.client.exec_command(cmd)
except Exception:
@@ -226,7 +192,7 @@ class TapeClient(object):
raise
else:
exitCode = stdout.channel.recv_exit_status()
- if not exitCode:
+ if exitCode == 0 or exitCode == 4:
self.logger.info("RECALL_CHECKSUM operation COMPLETED.")
else:
self.logger.error("RECALL_CHECKSUM operation FAILED.")
@@ -235,8 +201,7 @@ class TapeClient(object):
def disconnect(self):
"""Performs a cleanup and closes the connection."""
- self.taskList.clear()
- self.poolList.clear()
+ self.HSMFilesystemList.clear()
self.client.close()
def getSize(self, fsMountPoint):
@@ -261,7 +226,6 @@ class TapeClient(object):
#tc.connect()
#tc.copy("/home/curban/store/mydir", "/home/mydir")
#tc.copy("/home/curban/store/foo2.txt", "/home/mydir/foo2.txt")
-#tl = tc.getTaskList()
#fsSize = tc.getSize("/ia2_tape_stb_01")
#print(fsSize)
#tc.disconnect()
diff --git a/transfer_service/tape_pool.py b/transfer_service/tape_pool.py
deleted file mode 100644
index c73cc0ccc31ac7c3b16d8d52c01ee918a9fe23b1..0000000000000000000000000000000000000000
--- a/transfer_service/tape_pool.py
+++ /dev/null
@@ -1,136 +0,0 @@
-#!/usr/bin/env python
-#
-# This file is part of vospace-transfer-service
-# Copyright (C) 2021 Istituto Nazionale di Astrofisica
-# SPDX-License-Identifier: GPL-3.0-or-later
-#
-#
-#
-# Example of JSON object obtained with "eeadm pool list --json":
-#
-# {
-# "id": "c42ba7c9-1ec7-43a3-91af-b735a81ebbb2",
-# "name": "pl_generic_rw_01",
-# "media_restriction": "^.{8}$",
-# "capacity": 130982137036800,
-# "used_space": 61111860723712,
-# "free_space": 69870276313088,
-# "reclaimable_space": 263958922781,
-# "active_space": 60847901800931,
-# "non_appendable_space": 0,
-# "num_of_tapes": 15,
-# "format_class": "default",
-# "library_name": "tape_ia2_SA",
-# "library_id": "bc4bbe3f-b670-46e2-bba9-3fbc1fc2ce52",
-# "nodegroup_name": "G0",
-# "device_type": "LTO",
-# "worm": "no",
-# "fill_policy": "Default",
-# "owner": "System",
-# "mount_limit": 0,
-# "low_space_warning_enable": false,
-# "low_space_warning_threshold": 0,
-# "no_space_warning_enable": false,
-# "mode": "normal"
-# }
-#
-
-class TapePool(object):
-
- def __init__(self):
- self.id = None
- self.name = None
- self.mediaRestriction = None
- self.capacity = None
- self.usedSpace = None
- self.freeSpace = None
- self.reclaimableSpace = None
- self.activeSpace = None
- self.nonAppendableSpace = None
- self.numOfTapes = None
- self.formatClass = None
- self.libraryName = None
- self.libraryId = None
- self.nodeGroupName = None
- self.deviceType = None
- self.worm = None
- self.fillPolicy = None
- self.owner = None
- self.mountLimit = None
- self.lowSpaceWarningEnable = None
- self.lowSpaceWarningThreshold = None
- self.noSpaceWarningEnable = None
- self.mode = None
-
- """
- Getters
- """
-
- def getId(self):
- return self.id
-
- def getName(self):
- return self.name
-
- def getMediaRestriction(self):
- return self.mediaRestriction
-
- def getCapacity(self):
- return self.capacity
-
- def getUsedSpace(self):
- return self.usedSpace
-
- def getFreeSpace(self):
- return self.freeSpace
-
- def getReclaimableSpace(self):
- return self.reclaimableSpace
-
- def getActiveSpace(self):
- return self.activeSpace
-
- def getNonAppendableSpace(self):
- return self.nonAppendableSpace
-
- def getNumOfTapes(self):
- return self.numOfTapes
-
- def getFormatClass(self):
- return self.formatClass
-
- def getLibraryName(self):
- return self.libraryName
-
- def getLibraryId(self):
- return self.libraryId
-
- def getNodeGroupName(self):
- return self.nodeGroupName
-
- def getDeviceType(self):
- return self.deviceType
-
- def worm(self):
- return self.worm
-
- def getFillPolicy(self):
- return self.fillPolicy
-
- def getOwner(self):
- return self.owner
-
- def getMountLimit(self):
- return self.mountLimit
-
- def lowSpaceWarningEnable(self):
- return self.lowSpaceWarningEnable
-
- def getLowSpaceWarningThreshold(self):
- return self.lowSpaceWarningThreshold
-
- def noSpaceWarningEnable(self):
- return self.noSpaceWarningEnable
-
- def getMode(self):
- return self.mode
diff --git a/transfer_service/tape_task.py b/transfer_service/tape_task.py
deleted file mode 100644
index ed37b0516b640c182b4c3390accee2cfdd010451..0000000000000000000000000000000000000000
--- a/transfer_service/tape_task.py
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/usr/bin/env python
-#
-# This file is part of vospace-transfer-service
-# Copyright (C) 2021 Istituto Nazionale di Astrofisica
-# SPDX-License-Identifier: GPL-3.0-or-later
-#
-
-class TapeTask(object):
-
- def __init__(self):
- self.inUseTapes = None
- self.inUsePools = None
- self.inUseNodeGroups = None
- self.inUseDrives = None
- self.cmdParam = None
- self.result = None
- self.status = None
- self.completedTime = None
- self.startedTime = None
- self.createdTime = None
- self.setInUseLibs = None
- self.type = None
- self.taskId = None
- self.id = None
-
- """
- Getters
- """
-
- def getResult(self):
- return self.result
-
- def getStatus(self):
- return self.status
-
- def succeeded(self):
- return self.result == "succeeded"
-
- def failed(self):
- return self.result == "failed"
-
- def aborted(self):
- return self.result == "aborted"
-
- def canceled(self):
- return self.result == "canceled"
-
- def isCreated(self):
- return self.status == "created"
-
- def isRunning(self):
- return self.status == "running"
-
- def isWaiting(self):
- return self.status == "waiting"
-
- def isCompleted(self):
- return self.status == "completed"
-
-
- """
- Setters
- """
-
- def setInUseTapes(self, inUseTapes):
- self.inUseTapes = inUseTapes
-
- def setInUsePools(self, inUsePools):
- self.inUsePools = inUsePools
-
- def setInUseNodeGroups(self, inUseNodeGroups):
- self.inUseGroups = inUseGroups
-
- def setInUseDrives(self, inUseDrives):
- self.inUseNodeDrives = inUseDrives
-
- def setCmdParam(self, cmdParam):
- self.cmdParam = cmdParam
-
- def setResult(self, result):
- self.result = result
-
- def setStatus(self, status):
- self.status = status
-
- def setCompletedTime(self, completedTime):
- self.completedTime = completedTime
-
- def setStartedTime(self, startedTime):
- self.startedTime = startedTime
-
- def setCreatedTime(self, createdTime):
- self.createdTime = createdTime
-
- def setInUseLibs(self, inUseLibs):
- self.setInUseLibs = inUseLibs
-
- def setType(self, type):
- self.type = type
-
- def setTaskId(self, taskId):
- self.taskId = taskId
-
- def setId(self, id):
- self.id = id