Skip to content
Snippets Groups Projects
Commit 426744b8 authored by Nicola Fulvio Calabria's avatar Nicola Fulvio Calabria
Browse files

Follow links before sending their vos paths to file service archive

endpoint
parent 2aa4eefa
No related branches found
No related tags found
No related merge requests found
......@@ -10,6 +10,7 @@ import it.inaf.ia2.aa.data.User;
import it.inaf.oats.vospace.datamodel.Views;
import it.inaf.oats.vospace.exception.InvalidArgumentException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
......@@ -39,6 +40,9 @@ public class FileServiceClient {
@Autowired
private HttpServletRequest request;
@Autowired
private LinkService linkService;
public String startArchiveJob(Transfer transfer, String jobId) {
String target = transfer.getTarget().substring("vos://".length() + authority.length());
......@@ -64,6 +68,9 @@ public class FileServiceClient {
vosPaths.add(target);
}
// follow links to links in vosPaths
vosPaths = linkService.followLinksToLinks(vosPaths);
ArchiveRequest archiveRequest = new ArchiveRequest();
archiveRequest.setJobId(jobId);
archiveRequest.setPaths(vosPaths);
......
/*
* This file is part of vospace-rest
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.oats.vospace;
import it.inaf.oats.vospace.datamodel.NodeUtils;
import it.inaf.oats.vospace.exception.InternalFaultException;
import it.inaf.oats.vospace.persistence.NodeDAO;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import net.ivoa.xml.vospace.v2.LinkNode;
import net.ivoa.xml.vospace.v2.Node;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Service
public class LinkService {
@Value("${vospace-authority}")
private String authority;
@Value("${link-max-depth}")
private int linkMaxDepth;
@Autowired
private NodeDAO nodeDao;
// Returns a new list = the list in argument with paths of links to links are substituted with
// their actual destination vos path. Only links to external resources
// (http:// ... ) should remain
public List<String> followLinksToLinks(List<String> vosPaths) {
List<LinkNode> linkNodesInPaths = nodeDao.returnLinkNodesInList(vosPaths);
// No links no change
if(linkNodesInPaths.isEmpty())
return vosPaths;
List<String> linkVosPaths = linkNodesInPaths.stream()
.map(ln -> NodeUtils.getVosPath(ln)).collect(Collectors.toList());
// Safe copy of argument
List<String> resultVosPaths = new ArrayList<>(vosPaths);
resultVosPaths.removeAll(linkVosPaths);
// follow links and add resulting vos paths. The only remaining links
// are expected to be external (http://... and so on)
resultVosPaths.addAll(linkNodesInPaths.stream()
.map(ln -> NodeUtils.getVosPath(this.followLink(ln)))
.collect(Collectors.toList()));
return resultVosPaths;
}
public Node followLink(LinkNode linkNode) {
return this.followLinkRecursive(linkNode, 0);
}
private Node followLinkRecursive(LinkNode linkNode, int depth) {
if (depth >= linkMaxDepth) {
throw new InternalFaultException("Max link depth reached at link node: "
+ NodeUtils.getVosPath(linkNode));
}
String linkTarget = linkNode.getTarget();
if (URIUtils.isURIInternal(linkTarget)) {
String targetPath = URIUtils.returnVosPathFromNodeURI(linkTarget, authority);
Optional<Node> targetNodeOpt = nodeDao.listNode(targetPath);
Node targetNode = targetNodeOpt.orElseThrow(() -> new InternalFaultException("Broken Link to target: " + targetPath));
if (targetNode instanceof LinkNode) {
return this.followLinkRecursive(linkNode, ++depth);
} else {
return targetNode;
}
} else {
return linkNode;
}
}
}
......@@ -19,7 +19,7 @@ import it.inaf.oats.vospace.exception.NodeNotFoundException;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
import it.inaf.oats.vospace.exception.ProtocolNotSupportedException;
import it.inaf.oats.vospace.exception.NodeBusyException;
import it.inaf.oats.vospace.persistence.LinkedServiceDAO;
import it.inaf.oats.vospace.parent.persistence.LinkedServiceDAO;
import it.inaf.oats.vospace.persistence.LocationDAO;
import it.inaf.oats.vospace.persistence.NodeDAO;
import it.inaf.oats.vospace.persistence.model.Location;
......@@ -51,12 +51,12 @@ public class UriService {
@Value("${file-service-url}")
private String fileServiceUrl;
@Value("${link-max-depth}")
private int linkMaxDepth;
@Autowired
private NodeDAO nodeDao;
@Autowired
private LinkService linkService;
@Autowired
private LocationDAO locationDAO;
......@@ -158,7 +158,7 @@ public class UriService {
if (!NodeUtils.checkIfReadable(node, user.getName(), user.getGroups())) {
throw PermissionDeniedException.forPath(relativePath);
}
node = this.followLink((LinkNode) node);
node = linkService.followLink((LinkNode) node);
}
}
return node;
......@@ -314,35 +314,6 @@ public class UriService {
return (Transfer) job.getJobInfo().getAny().get(0);
}
private Node followLink(LinkNode linkNode) {
return this.followLinkRecursive(linkNode, 0);
}
private Node followLinkRecursive(LinkNode linkNode, int depth) {
if (depth >= linkMaxDepth) {
throw new InternalFaultException("Max link depth reached at link node: "
+ NodeUtils.getVosPath(linkNode));
}
String linkTarget = linkNode.getTarget();
if (URIUtils.isURIInternal(linkTarget)) {
String targetPath = URIUtils.returnVosPathFromNodeURI(linkTarget, authority);
Optional<Node> targetNodeOpt = nodeDao.listNode(targetPath);
Node targetNode = targetNodeOpt.orElseThrow(() -> new InternalFaultException("Broken Link to target: " + targetPath));
if (targetNode instanceof LinkNode) {
return this.followLinkRecursive(linkNode, ++depth);
} else {
return targetNode;
}
} else {
return linkNode;
}
}
public enum ProtocolType {
// Please keep the URIs in this enum UNIQUE!
// added a unit test to check this
......
/*
* This file is part of vospace-rest
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.oats.vospace.persistence;
import com.fasterxml.jackson.databind.ObjectMapper;
import javax.sql.DataSource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Repository
public class LinkedServiceDAO {
private static final Logger LOG = LoggerFactory.getLogger(LinkedServiceDAO.class);
private static final ObjectMapper MAPPER = new ObjectMapper();
private final JdbcTemplate jdbcTemplate;
@Autowired
public LinkedServiceDAO(DataSource dataSource) {
jdbcTemplate = new JdbcTemplate(dataSource);
}
public boolean isLinkedServiceUrl(String targetUrl) {
String sql = " SELECT COUNT(*) > 0\n"
+ "FROM linked_service\n"
+ "WHERE ? LIKE service_base_url || '%'";
return jdbcTemplate.query(sql, ps -> {
ps.setString(1, targetUrl);
}, row -> {
if (!row.next()) {
throw new IllegalStateException("Expected one result");
}
return row.getBoolean(1);
});
}
}
......@@ -18,6 +18,7 @@ import java.sql.SQLException;
import java.sql.Types;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
......@@ -552,6 +553,36 @@ public class NodeDAO {
}
}
public List<LinkNode> returnLinkNodesInList(List<String> vosPaths) {
if (vosPaths.isEmpty()) {
throw new IllegalArgumentException("Received empty list of paths");
}
String sql = "SELECT n.node_id, get_vos_path(n.node_id) as vos_path, n.name,\n"
+ "n.type, n.async_trans, n.sticky, n.job_id IS NOT NULL AS busy_state, n.creator_id, n.group_read, n.group_write,\n"
+ "n.is_public, n.content_length, n.created_on, n.last_modified, n.accept_views, n.provide_views, n.quota, n.content_md5, n.target\n"
+ "FROM node n\n"
+ "WHERE " + String.join(" OR ", Collections.nCopies(vosPaths.size(), "n.node_id = id_from_vos_path(?)")) + "\n"
+ "AND n.type = 'link'\n";
return jdbcTemplate.query(conn -> {
PreparedStatement ps = conn.prepareStatement(sql);
int i = 0;
for (String vosPath : vosPaths) {
ps.setString(++i, vosPath);
}
return ps;
}, rs -> {
List<LinkNode> linkNodes = new ArrayList<>();
while (rs.next()) {
linkNodes.add((LinkNode) this.getNodeFromResultSet(rs));
}
return linkNodes;
});
}
private String getGroupsString(ResultSet rs, String column) throws SQLException {
Array array = rs.getArray(column);
if (array == null) {
......
......@@ -16,7 +16,7 @@ import it.inaf.oats.vospace.exception.InvalidArgumentException;
import it.inaf.oats.vospace.exception.NodeBusyException;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
import it.inaf.oats.vospace.exception.ProtocolNotSupportedException;
import it.inaf.oats.vospace.persistence.LinkedServiceDAO;
import it.inaf.oats.vospace.parent.persistence.LinkedServiceDAO;
import it.inaf.oats.vospace.persistence.LocationDAO;
import it.inaf.oats.vospace.persistence.NodeDAO;
import it.inaf.oats.vospace.persistence.model.Location;
......
/*
* This file is part of vospace-rest
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.oats.vospace.persistence;
import javax.sql.DataSource;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import static org.junit.jupiter.api.Assertions.assertFalse;
@ExtendWith(SpringExtension.class)
@ContextConfiguration(classes = {DataSourceConfig.class})
@TestPropertySource(locations = "classpath:test.properties")
public class LinkedServiceDAOTest {
@Autowired
private DataSource dataSource;
private LinkedServiceDAO dao;
@BeforeEach
public void init() {
dao = new LinkedServiceDAO(dataSource);
}
@Test
void testIsLinkedService() {
assertTrue(dao.isLinkedServiceUrl("http://archives.ia2.inaf.it/files/aao/pippofile.fits.gz"));
assertFalse(dao.isLinkedServiceUrl("http://noportal.ia2.inaf.it/files/nop/nopippofile.tar.gz"));
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment