From 7e8b01181910da200145313eb4f9d60a5a144b69 Mon Sep 17 00:00:00 2001
From: Nicola Fulvio Calabria <nicola.calabria@inaf.it>
Date: Sun, 24 Oct 2021 21:32:43 +0200
Subject: [PATCH] Added link support to archives
---
.../controller/ArchiveFileController.java | 6 +-
.../ia2/transfer/persistence/FileDAO.java | 15 ++-
.../transfer/persistence/model/FileInfo.java | 11 ++-
.../ia2/transfer/service/ArchiveService.java | 96 ++++++++++++++++---
.../controller/ArchiveFileControllerTest.java | 2 +-
.../transfer/service/ArchiveServiceTest.java | 12 ++-
6 files changed, 117 insertions(+), 25 deletions(-)
diff --git a/src/main/java/it/inaf/ia2/transfer/controller/ArchiveFileController.java b/src/main/java/it/inaf/ia2/transfer/controller/ArchiveFileController.java
index 755c529..f69ba9b 100644
--- a/src/main/java/it/inaf/ia2/transfer/controller/ArchiveFileController.java
+++ b/src/main/java/it/inaf/ia2/transfer/controller/ArchiveFileController.java
@@ -12,6 +12,7 @@ import it.inaf.ia2.transfer.service.ArchiveService;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
import java.io.File;
import java.util.concurrent.CompletableFuture;
+import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpHeaders;
@@ -29,6 +30,9 @@ public class ArchiveFileController extends AuthenticatedFileController {
@Autowired
private ArchiveService archiveService;
+
+ @Autowired
+ private HttpServletRequest servletRequest;
@Autowired
private HttpServletResponse response;
@@ -45,7 +49,7 @@ public class ArchiveFileController extends AuthenticatedFileController {
job.setVosPaths(archiveRequest.getPaths());
CompletableFuture.runAsync(() -> {
- handleFileJob(() -> archiveService.createArchive(job), job.getJobId());
+ handleFileJob(() -> archiveService.createArchive(job, servletRequest), job.getJobId());
});
HttpHeaders headers = new HttpHeaders();
diff --git a/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java b/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java
index cff63fa..b04847b 100644
--- a/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java
+++ b/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java
@@ -47,7 +47,7 @@ public class FileDAO {
+ "accept_views, provide_views, l.location_type, n.path <> n.relative_path AS virtual_parent,\n"
+ "(SELECT user_name FROM users WHERE user_id = creator_id) AS username, n.job_id,\n"
+ "base_path, get_os_path(n.node_id) AS os_path, ? AS vos_path, false AS is_directory,\n"
- + "type = 'link' AS is_link,\n"
+ + "n.type = 'link' AS is_link, n.target,\n"
+ "fs_path \n"
+ "FROM node n\n"
+ "JOIN location l ON (n.location_id IS NOT NULL AND n.location_id = l.location_id) OR (n.location_id IS NULL AND l.location_id = ?)\n"
@@ -180,7 +180,7 @@ public class FileDAO {
+ "(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n"
+ "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"
+ "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n"
- + "n.type = 'link' AS is_link, l.location_type\n"
+ + "n.type = 'link' AS is_link, n.target, l.location_type\n"
+ "FROM node n\n"
+ "JOIN node p ON p.path @> n.path\n"
+ "LEFT JOIN location l ON l.location_id = n.location_id\n"
@@ -213,7 +213,7 @@ public class FileDAO {
+ "(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n"
+ "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"
+ "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n"
- + "n.type = 'link' AS is_link, l.location_type\n"
+ + "n.type = 'link' AS is_link, n.target, l.location_type\n"
+ "FROM node n\n"
+ "JOIN node p ON p.path @> n.path\n"
+ "LEFT JOIN location l ON l.location_id = n.location_id\n"
@@ -289,11 +289,16 @@ public class FileDAO {
long contentLength = rs.getLong("content_length");
if (!rs.wasNull()) {
fi.setContentLength(contentLength);
- }
+ }
fi.setContentMd5(rs.getString("content_md5"));
fi.setContentType(rs.getString("content_type"));
fi.setDirectory(rs.getBoolean("is_directory"));
- fi.setLink(rs.getBoolean("is_link"));
+ if(rs.getBoolean("is_link")){
+ fi.setLink(true);
+ fi.setTarget(rs.getString("target"));
+ } else {
+ fi.setLink(false);
+ }
fi.setJobId(rs.getString("job_id"));
int locationId = rs.getInt("location_id");
if (!rs.wasNull()) {
diff --git a/src/main/java/it/inaf/ia2/transfer/persistence/model/FileInfo.java b/src/main/java/it/inaf/ia2/transfer/persistence/model/FileInfo.java
index f0b032d..4081d07 100644
--- a/src/main/java/it/inaf/ia2/transfer/persistence/model/FileInfo.java
+++ b/src/main/java/it/inaf/ia2/transfer/persistence/model/FileInfo.java
@@ -17,11 +17,12 @@ public class FileInfo {
private String fsPath;
// actualBasePath differs from base path in db due to some location type
// dependent manipulations (performed by FileDAO)
- private String actualBasePath;
+ private String actualBasePath;
private boolean isPublic;
private boolean virtualParent;
private boolean directory;
private boolean link;
+ private String target;
private List<String> groupRead;
private List<String> groupWrite;
private String ownerId;
@@ -36,6 +37,14 @@ public class FileInfo {
private String locationType;
private String jobId;
+ public String getTarget() {
+ return target;
+ }
+
+ public void setTarget(String target) {
+ this.target = target;
+ }
+
public int getNodeId() {
return nodeId;
}
diff --git a/src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java b/src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java
index 2a98e89..a273d77 100644
--- a/src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java
+++ b/src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java
@@ -5,6 +5,9 @@
*/
package it.inaf.ia2.transfer.service;
+import it.inaf.ia2.aa.ServletRapClient;
+import it.inaf.ia2.aa.data.User;
+import it.inaf.ia2.rap.client.call.TokenExchangeRequest;
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.FileDAO;
import it.inaf.ia2.transfer.persistence.JobDAO;
@@ -13,6 +16,7 @@ import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.oats.vospace.exception.InternalFaultException;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
import it.inaf.oats.vospace.exception.QuotaExceededException;
+import it.inaf.oats.vospace.parent.persistence.LinkedServiceDAO;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -28,6 +32,7 @@ import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.annotation.PostConstruct;
+import javax.servlet.http.HttpServletRequest;
import net.ivoa.xml.uws.v1.ExecutionPhase;
import org.kamranzafar.jtar.TarEntry;
import org.kamranzafar.jtar.TarOutputStream;
@@ -52,6 +57,9 @@ public class ArchiveService {
@Autowired
private LocationDAO locationDAO;
+
+ @Autowired
+ private LinkedServiceDAO linkedServiceDAO;
@Autowired
private JobDAO jobDAO;
@@ -61,6 +69,9 @@ public class ArchiveService {
@Autowired
private RestTemplate restTemplate;
+
+ @Autowired
+ private ServletRapClient rapClient;
@Value("${upload_location_id}")
private int uploadLocationId;
@@ -84,7 +95,7 @@ public class ArchiveService {
}
}
- public <O extends OutputStream, E> void createArchive(ArchiveJob job) {
+ public <O extends OutputStream, E> void createArchive(ArchiveJob job, HttpServletRequest servletRequest) {
jobDAO.updateJobPhase(ExecutionPhase.EXECUTING, job.getJobId());
@@ -101,7 +112,7 @@ public class ArchiveService {
// it will be initialized only when necessary
Map<Integer, String> portalLocationUrls = null;
- try ( ArchiveHandler<O, E> handler = getArchiveHandler(archiveFile, job.getType())) {
+ try (ArchiveHandler<O, E> handler = getArchiveHandler(archiveFile, job.getType())) {
for (FileInfo fileInfo : fileDAO.getArchiveFileInfos(job.getVosPaths())) {
@@ -112,13 +123,21 @@ public class ArchiveService {
continue;
}
+ // I expect only external links
+ // local links have been resolved before calling this endpoint
+ if (fileInfo.isLink()) {
+ downloadExternalLinkIntoArchive(fileInfo, relPath,
+ job.getPrincipal(), handler, servletRequest);
+ continue;
+ }
+
if (fileInfo.getLocationId() != null && "portal".equals(fileInfo.getLocationType())) {
// remote file
if (portalLocationUrls == null) {
portalLocationUrls = locationDAO.getPortalLocationUrls();
}
String url = portalLocationUrls.get(fileInfo.getLocationId());
- downloadFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler, url);
+ downloadRemoteLocationFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler, url);
} else {
// local file or virtual directory
writeFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler);
@@ -272,15 +291,7 @@ public class ArchiveService {
}
}
- private <O extends OutputStream, E> void downloadFileIntoArchive(FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler, String baseUrl) {
-
- if (baseUrl == null) {
- LOG.error("Location URL not found for location " + fileInfo.getLocationId());
- throw new InternalFaultException("Unable to retrieve location of file " + fileInfo.getVirtualPath());
- }
-
- String url = baseUrl + "/" + fileInfo.getVirtualName();
-
+ private <O extends OutputStream, E> void downloadFromUrlIntoArchive(String url, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler) {
LOG.trace("Downloading file from " + url);
restTemplate.execute(url, HttpMethod.GET, req -> {
@@ -290,10 +301,10 @@ public class ArchiveService {
}
}, res -> {
File tmpFile = Files.createTempFile("download", null).toFile();
- try ( FileOutputStream os = new FileOutputStream(tmpFile)) {
+ try (FileOutputStream os = new FileOutputStream(tmpFile)) {
res.getBody().transferTo(os);
handler.putNextEntry(tmpFile, relPath);
- try ( FileInputStream is = new FileInputStream(tmpFile)) {
+ try (FileInputStream is = new FileInputStream(tmpFile)) {
is.transferTo(handler.getOutputStream());
}
} finally {
@@ -303,6 +314,43 @@ public class ArchiveService {
}, new Object[]{});
}
+ private <O extends OutputStream, E> void downloadRemoteLocationFileIntoArchive(
+ FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,
+ ArchiveHandler<O, E> handler, String baseUrl) {
+
+ if (baseUrl == null) {
+ LOG.error("Location URL not found for location " + fileInfo.getLocationId());
+ throw new InternalFaultException("Unable to retrieve location of file "
+ + fileInfo.getVirtualPath());
+ }
+
+ String url = baseUrl + "/" + fileInfo.getVirtualName();
+
+ downloadFromUrlIntoArchive(url, relPath, tokenPrincipal, handler);
+ }
+
+ private <O extends OutputStream, E> void downloadExternalLinkIntoArchive(
+ FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,
+ ArchiveHandler<O, E> handler, HttpServletRequest servletRequest) {
+
+ String url = fileInfo.getTarget();
+
+ if (url == null || url.isBlank()) {
+ LOG.error("Target URL of link at path: {} is null or blank", fileInfo.getVirtualPath());
+ throw new InternalFaultException("Target URL of link at path: "
+ + fileInfo.getVirtualPath() + " is null or blank");
+ }
+
+ // Append token if url is recognized
+
+ if (linkedServiceDAO.isLinkedServiceUrl(url)) {
+ url += "?token=" + getEndpointToken(tokenPrincipal, url, servletRequest);
+ }
+
+ downloadFromUrlIntoArchive(url, relPath, tokenPrincipal, handler);
+
+ }
+
private <O extends OutputStream, E> void writeFileIntoArchive(FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler) throws IOException {
if (!authorizationService.isDownloadable(fileInfo, tokenPrincipal)) {
throw PermissionDeniedException.forPath(fileInfo.getVirtualPath());
@@ -311,9 +359,27 @@ public class ArchiveService {
File file = new File(fileInfo.getFilePath());
LOG.trace("Adding file " + file.getAbsolutePath() + " to tar archive");
- try ( InputStream is = new FileInputStream(file)) {
+ try (InputStream is = new FileInputStream(file)) {
handler.putNextEntry(file, relPath);
is.transferTo(handler.getOutputStream());
}
}
+
+ private String getEndpointToken(TokenPrincipal tokenPrincipal,
+ String endpoint, HttpServletRequest servletRequest) {
+
+ String token = tokenPrincipal.getToken();
+
+ if (token == null) {
+ throw new PermissionDeniedException("Token is null");
+ }
+
+ TokenExchangeRequest exchangeRequest = new TokenExchangeRequest()
+ .setSubjectToken(token)
+ .setResource(endpoint);
+
+ // TODO: add audience and scope
+ return rapClient.exchangeToken(exchangeRequest, servletRequest);
+ }
+
}
diff --git a/src/test/java/it/inaf/ia2/transfer/controller/ArchiveFileControllerTest.java b/src/test/java/it/inaf/ia2/transfer/controller/ArchiveFileControllerTest.java
index b5d04fd..764322f 100644
--- a/src/test/java/it/inaf/ia2/transfer/controller/ArchiveFileControllerTest.java
+++ b/src/test/java/it/inaf/ia2/transfer/controller/ArchiveFileControllerTest.java
@@ -65,7 +65,7 @@ public class ArchiveFileControllerTest {
assertEquals("user1", job.getPrincipal().getName());
assertEquals(2, job.getVosPaths().size());
return true;
- }));
+ }), any());
}
@Test
diff --git a/src/test/java/it/inaf/ia2/transfer/service/ArchiveServiceTest.java b/src/test/java/it/inaf/ia2/transfer/service/ArchiveServiceTest.java
index 53f7bc9..081e1ac 100644
--- a/src/test/java/it/inaf/ia2/transfer/service/ArchiveServiceTest.java
+++ b/src/test/java/it/inaf/ia2/transfer/service/ArchiveServiceTest.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.function.Function;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
+import javax.servlet.http.HttpServletRequest;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -69,6 +70,9 @@ public class ArchiveServiceTest {
@MockBean
private RestTemplate restTemplate;
+
+ @MockBean
+ private HttpServletRequest servletRequest;
@MockBean
private AuthorizationService authorizationService;
@@ -139,6 +143,8 @@ public class ArchiveServiceTest {
job.setJobId("job2");
job.setType(ArchiveJob.Type.ZIP);
job.setVosPaths(Arrays.asList("/ignore"));
+
+ when(servletRequest.getUserPrincipal()).thenReturn(job.getPrincipal());
File user2Dir = tmpDir.toPath().resolve("user2").toFile();
user2Dir.mkdir();
@@ -153,7 +159,7 @@ public class ArchiveServiceTest {
}
Assertions.assertThrows(QuotaExceededException.class, () -> {
- archiveService.createArchive(job);
+ archiveService.createArchive(job, servletRequest);
});
}
private static abstract class TestArchiveHandler<I extends InputStream, E> {
@@ -193,6 +199,8 @@ public class ArchiveServiceTest {
job.setJobId("abcdef");
job.setType(type);
job.setVosPaths(Arrays.asList(parent + "/dir1", parent + "/dir2", parent + "/file6"));
+
+ when(servletRequest.getUserPrincipal()).thenReturn(job.getPrincipal());
when(authorizationService.isDownloadable(any(), any())).thenReturn(true);
@@ -224,7 +232,7 @@ public class ArchiveServiceTest {
}).when(restTemplate).execute(eq("http://portal/base/url/portal-file"), eq(HttpMethod.GET),
any(RequestCallback.class), any(ResponseExtractor.class), any(Object[].class));
- archiveService.createArchive(job);
+ archiveService.createArchive(job, servletRequest);
File result = tmpDir.toPath().resolve("user1").resolve("abcdef." + extension).toFile();
--
GitLab