From 209bdccfdf6cf03524a712b91cc0d7a956747e60 Mon Sep 17 00:00:00 2001
From: Sonia Zorba <sonia.zorba@inaf.it>
Date: Fri, 10 Sep 2021 16:49:08 +0200
Subject: [PATCH] Added location type to FileInfo and fixed issue with
generation of archives containing async nodes
---
.../ia2/transfer/persistence/FileDAO.java | 29 +++++++--------
.../transfer/persistence/model/FileInfo.java | 15 ++++++--
.../ia2/transfer/service/ArchiveService.java | 2 +-
.../ia2/transfer/service/FileCopyService.java | 36 +++++++++----------
.../ia2/transfer/persistence/FileDAOTest.java | 4 +++
5 files changed, 50 insertions(+), 36 deletions(-)
diff --git a/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java b/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java
index 7a54147..0c006a6 100644
--- a/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java
+++ b/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java
@@ -94,7 +94,7 @@ public class FileDAO {
// This should never happen thanks to database constraints
throw new IllegalStateException("More than one node at path: " + vosPath);
}
-
+
return nodes;
}
@@ -175,8 +175,8 @@ public class FileDAO {
+ "n.accept_views, n.provide_views, l.location_type, n.path <> n.relative_path AS virtual_parent,\n"
+ "(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n"
+ "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"
- + "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id\n"
- + "n.type = 'link' AS is_link\n"
+ + "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n"
+ + "n.type = 'link' AS is_link, l.location_type\n"
+ "FROM node n\n"
+ "JOIN node p ON p.path @> n.path\n"
+ "LEFT JOIN location l ON l.location_id = n.location_id\n"
@@ -200,16 +200,16 @@ public class FileDAO {
});
}
- // TODO: same problem as get archive file infos
+ // TODO: same problem as get archive file infos
public List<FileInfo> getBranchFileInfos(String rootVosPath, String jobId) {
-
+
String sql = "SELECT n.node_id, n.is_public, n.group_read, n.group_write, n.creator_id, n.async_trans,\n"
+ "n.content_type, n.content_encoding, n.content_length, n.content_md5,\n"
+ "n.accept_views, n.provide_views, l.location_type, n.path <> n.relative_path AS virtual_parent,\n"
+ "(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n"
+ "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"
+ "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n"
- + "n.type = 'link' AS is_link\n"
+ + "n.type = 'link' AS is_link, l.location_type\n"
+ "FROM node n\n"
+ "JOIN node p ON p.path @> n.path\n"
+ "LEFT JOIN location l ON l.location_id = n.location_id\n"
@@ -232,17 +232,17 @@ public class FileDAO {
}
public void setBranchLocationId(String rootVosPath, String jobId, int locationId) {
-
+
String cte = "SELECT n.node_id AS id\n"
+ "FROM node n\n"
+ "JOIN node p ON p.path @> n.path\n"
+ "WHERE (p.node_id = id_from_vos_path(?) AND n.job_id = ?)\n";
-
+
String update = "UPDATE node\n"
+ " SET location_id = ?\n"
+ "FROM cte\n"
+ "WHERE node_id = cte.id\n";
-
+
String sql = "WITH cte AS (\n"
+ cte
+ ")\n"
@@ -256,16 +256,16 @@ public class FileDAO {
return ps;
});
}
-
+
// !! duplicate code from NodeDAO
public void releaseBusyNodesByJobId(String jobId) {
String sql = "UPDATE node SET job_id = NULL WHERE job_id = ?";
-
+
jdbcTemplate.update(conn -> {
PreparedStatement ps = conn.prepareStatement(sql);
- ps.setString(1, jobId);
+ ps.setString(1, jobId);
return ps;
- });
+ });
}
private FileInfo getFileInfo(ResultSet rs) throws SQLException {
@@ -290,10 +290,11 @@ public class FileDAO {
fi.setContentType(rs.getString("content_type"));
fi.setDirectory(rs.getBoolean("is_directory"));
fi.setLink(rs.getBoolean("is_link"));
- fi.setJobId(rs.getString("job_id"));
+ fi.setJobId(rs.getString("job_id"));
int locationId = rs.getInt("location_id");
if (!rs.wasNull()) {
fi.setLocationId(locationId);
+ fi.setLocationType(rs.getString("location_type"));
}
fillOsPath(fi, rs);
diff --git a/src/main/java/it/inaf/ia2/transfer/persistence/model/FileInfo.java b/src/main/java/it/inaf/ia2/transfer/persistence/model/FileInfo.java
index 49f859e..7396e6c 100644
--- a/src/main/java/it/inaf/ia2/transfer/persistence/model/FileInfo.java
+++ b/src/main/java/it/inaf/ia2/transfer/persistence/model/FileInfo.java
@@ -28,6 +28,7 @@ public class FileInfo {
private Long contentLength;
private String contentMd5;
private Integer locationId;
+ private String locationType;
private String jobId;
public int getNodeId() {
@@ -109,7 +110,7 @@ public class FileInfo {
public void setDirectory(boolean directory) {
this.directory = directory;
}
-
+
public boolean isLink() {
return link;
}
@@ -182,6 +183,14 @@ public class FileInfo {
this.locationId = locationId;
}
+ public String getLocationType() {
+ return locationType;
+ }
+
+ public void setLocationType(String locationType) {
+ this.locationType = locationType;
+ }
+
public String getJobId() {
return jobId;
}
@@ -189,8 +198,8 @@ public class FileInfo {
public void setJobId(String jobId) {
this.jobId = jobId;
}
-
+
public static String getVosParentPath(FileInfo fileInfo) {
- return fileInfo.getVirtualPath().substring(0, fileInfo.getVirtualPath().lastIndexOf("/"));
+ return fileInfo.getVirtualPath().substring(0, fileInfo.getVirtualPath().lastIndexOf("/"));
}
}
diff --git a/src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java b/src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java
index 74a1fd1..f124493 100644
--- a/src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java
+++ b/src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java
@@ -112,7 +112,7 @@ public class ArchiveService {
continue;
}
- if (fileInfo.getLocationId() != null && fileInfo.getLocationId() != uploadLocationId) {
+ if (fileInfo.getLocationId() != null && "portal".equals(fileInfo.getLocationType())) {
// remote file
if (portalLocationUrls == null) {
portalLocationUrls = locationDAO.getPortalLocationUrls();
diff --git a/src/main/java/it/inaf/ia2/transfer/service/FileCopyService.java b/src/main/java/it/inaf/ia2/transfer/service/FileCopyService.java
index b18e684..23b565a 100644
--- a/src/main/java/it/inaf/ia2/transfer/service/FileCopyService.java
+++ b/src/main/java/it/inaf/ia2/transfer/service/FileCopyService.java
@@ -43,32 +43,32 @@ public class FileCopyService {
@Autowired
private RestTemplate restTemplate;
-
+
@Autowired
private PutFileService putFileService;
-
+
@Value("${upload_location_id}")
private int uploadLocationId;
public void copyFiles(String sourceRootVosPath,
String destinationRootVosPath, String jobId, TokenPrincipal principal) {
-
- LOG.trace("copyFiles called for source {}, destination {}, from jobId \"{}\"",
+
+ LOG.trace("copyFiles called for source {}, destination {}, from jobId \"{}\"",
sourceRootVosPath, destinationRootVosPath, jobId);
// We use jobId to identify nodes created by the REST part of CopyNode
// We expect them to be locked
List<FileInfo> sources
= fileDAO.getBranchFileInfos(sourceRootVosPath, jobId);
-
+
LOG.debug("found {} sources", sources.size());
if (sources.isEmpty()) {
throw new NodeNotFoundException(sourceRootVosPath);
}
-
-
- // Set location of destinations to this file service update location
+
+
+ // Set location of destinations to this file service update location
// before retrieving file infos
fileDAO.setBranchLocationId(destinationRootVosPath, jobId, uploadLocationId);
@@ -76,7 +76,7 @@ public class FileCopyService {
= fileDAO.getBranchFileInfos(destinationRootVosPath, jobId);
LOG.debug("found {} destinations", destinations.size());
-
+
if (destinations.isEmpty()) {
throw new NodeNotFoundException(destinationRootVosPath);
}
@@ -94,7 +94,7 @@ public class FileCopyService {
principal);
}
-
+
private void fillDestinations(List<FileInfo> sourcesFileInfos,
List<FileInfo> destinationFileInfos,
String sourceRootVosPath,
@@ -102,7 +102,7 @@ public class FileCopyService {
TokenPrincipal principal) {
// it will be initialized only when necessary
- Map<Integer, String> portalLocationUrls = null;
+ Map<Integer, String> portalLocationUrls = null;
for (FileInfo destinationFileInfo : destinationFileInfos) {
LOG.trace("Processing {} destination", destinationFileInfo.getVirtualPath());
@@ -134,25 +134,25 @@ public class FileCopyService {
}
}
- if (sourceFileInfo.getLocationId() != null && sourceFileInfo.getLocationId() != uploadLocationId) {
+ if (sourceFileInfo.getLocationId() != null && "portal".equals(sourceFileInfo.getLocationType())) {
// remote file
if (portalLocationUrls == null) {
portalLocationUrls = locationDAO.getPortalLocationUrls();
}
String url = portalLocationUrls.get(sourceFileInfo.getLocationId());
// download file to destination disk path
- this.downloadFileToDisk(sourceFileInfo,
- destinationFileInfo,
+ this.downloadFileToDisk(sourceFileInfo,
+ destinationFileInfo,
principal, url, remainingQuota);
} else {
- // local file
+ // local file
this.copyLocalFile(sourceFileInfo, destinationFileInfo, principal, remainingQuota);
}
}
- }
+ }
}
private String getCorrespondingSourceVosPath(String sourceRootVosPath,
@@ -200,7 +200,7 @@ public class FileCopyService {
private void copyLocalFile(FileInfo sourceFileInfo,
FileInfo destinationFileInfo, TokenPrincipal tokenPrincipal, Long remainingQuota) {
-
+
// Check permission
if (!authorizationService.isDownloadable(sourceFileInfo, tokenPrincipal)) {
throw PermissionDeniedException.forPath(sourceFileInfo.getVirtualPath());
@@ -208,7 +208,7 @@ public class FileCopyService {
File file = new File(sourceFileInfo.getOsPath());
LOG.trace("Copying file: {} to {}",file.getAbsolutePath(), destinationFileInfo.getOsPath());
-
+
putFileService.copyLocalFile(sourceFileInfo, destinationFileInfo, remainingQuota);
}
diff --git a/src/test/java/it/inaf/ia2/transfer/persistence/FileDAOTest.java b/src/test/java/it/inaf/ia2/transfer/persistence/FileDAOTest.java
index 622f5dd..b8336a0 100644
--- a/src/test/java/it/inaf/ia2/transfer/persistence/FileDAOTest.java
+++ b/src/test/java/it/inaf/ia2/transfer/persistence/FileDAOTest.java
@@ -88,10 +88,14 @@ public class FileDAOTest {
assertEquals("/home/vospace/upload/user1/file1", fileInfos.get(0).getOsPath());
assertEquals("/home/vospace/upload/user1/file2", fileInfos.get(1).getOsPath());
+ assertEquals("user", fileInfos.get(1).getLocationType());
assertTrue(fileInfos.get(2).isDirectory());
+ assertNull(fileInfos.get(2).getLocationType());
assertEquals("/home/username1/retrieve/subdir1/file3", fileInfos.get(3).getOsPath());
assertEquals("/home/username1/retrieve/subdir1/file4", fileInfos.get(4).getOsPath());
+ assertEquals("async", fileInfos.get(4).getLocationType());
assertEquals("portal-file", fileInfos.get(5).getVirtualName());
+ assertEquals("portal", fileInfos.get(5).getLocationType());
}
@Test
--
GitLab