Skip to content
Snippets Groups Projects
Commit 1a6532f7 authored by Nicola Fulvio Calabria's avatar Nicola Fulvio Calabria
Browse files

Commented out or deleted code about portal locations

parent 2e7e286c
No related branches found
No related tags found
No related merge requests found
/*
* This file is part of vospace-file-service
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.ia2.transfer.persistence;
import java.util.HashMap;
import java.util.Map;
import javax.sql.DataSource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
@Repository
public class LocationDAO {
private final JdbcTemplate jdbcTemplate;
@Autowired
public LocationDAO(DataSource fileCatalogDatasource) {
this.jdbcTemplate = new JdbcTemplate(fileCatalogDatasource);
}
public Map<Integer, String> getPortalLocationUrls() {
String sql = "SELECT location_id, hostname, base_url\n"
+ "FROM location l\n"
+ "JOIN storage s ON s.storage_id = l.storage_dest_id\n"
+ "WHERE location_type = 'portal'";
return jdbcTemplate.query(sql, rs -> {
Map<Integer, String> locationUrls = new HashMap<>();
while (rs.next()) {
int locationId = rs.getInt("location_id");
String hostname = rs.getString("hostname");
String baseUrl = rs.getString("base_url");
String url = "http://" + hostname + baseUrl;
locationUrls.put(locationId, url);
}
return locationUrls;
});
}
}
......@@ -6,12 +6,10 @@
package it.inaf.ia2.transfer.service;
import it.inaf.ia2.aa.ServletRapClient;
import it.inaf.ia2.aa.data.User;
import it.inaf.ia2.rap.client.call.TokenExchangeRequest;
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.FileDAO;
import it.inaf.ia2.transfer.persistence.JobDAO;
import it.inaf.ia2.transfer.persistence.LocationDAO;
import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.oats.vospace.exception.InternalFaultException;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
......@@ -57,9 +55,6 @@ public class ArchiveService {
@Autowired
private FileDAO fileDAO;
@Autowired
private LocationDAO locationDAO;
@Autowired
private LinkedServiceDAO linkedServiceDAO;
......@@ -75,9 +70,6 @@ public class ArchiveService {
@Autowired
private ServletRapClient rapClient;
@Value("${upload_location_id}")
private int uploadLocationId;
// Directory containing temporary files generated by jobs.
@Value("${generated.dir}")
private String generatedDirString;
......@@ -109,13 +101,11 @@ public class ArchiveService {
List<ArchiveEntryDescriptor> entryDescriptors = job.getEntryDescriptors();
String commonParent = getCommonParent(entryDescriptors);
// support directory used to generate folder inside tar files (path is redefined each time by TarEntry class)
File supportDir = Files.createTempDirectory("dir").toFile();
try (ArchiveHandler<O, E> handler = getArchiveHandler(archiveFile, job.getType())) {
fillArchive(entryDescriptors, commonParent, supportDir,
fillArchive(entryDescriptors, supportDir,
job.getPrincipal(), servletRequest, handler);
} finally {
FileSystemUtils.deleteRecursively(supportDir);
......@@ -127,13 +117,10 @@ public class ArchiveService {
}
private <O extends OutputStream, E> void fillArchive(
List<ArchiveEntryDescriptor> entryDescriptors, String commonParent,
List<ArchiveEntryDescriptor> entryDescriptors,
File supportDir, TokenPrincipal tokenPrincipal,
HttpServletRequest servletRequest, ArchiveHandler<O, E> handler) throws IOException {
// it will be initialized only when necessary
Map<Integer, String> portalLocationUrls = null;
List<ArchiveEntryDescriptor> noTargetEntryDescriptors
= entryDescriptors.stream().filter(ed -> !ed.isPointingToAnotherNode())
.collect(Collectors.toList());
......@@ -143,12 +130,14 @@ public class ArchiveService {
.map(ed -> ed.getVosPath())
.collect(Collectors.toList());
String commonParent = getCommonParent(entryDescriptors);
if (!vosPaths.isEmpty()) {
for (FileInfo fileInfo : fileDAO.getArchiveFileInfos(vosPaths)) {
String relPath = fileInfo.getVirtualPath().substring(commonParent.length());
this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, portalLocationUrls, servletRequest, handler);
this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, servletRequest, handler);
}
}
......@@ -175,7 +164,7 @@ public class ArchiveService {
for (String vosPath : linkVosPaths) {
String relPath = vosPath.substring(commonParent.length());
this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, portalLocationUrls, servletRequest, handler);
this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, servletRequest, handler);
}
}
}
......@@ -183,8 +172,7 @@ public class ArchiveService {
private <O extends OutputStream, E> void insertEntryIntoArchive(
FileInfo fileInfo, File supportDir, String relPath,
TokenPrincipal tokenPrincipal, Map<Integer, String> portalLocationUrls,
HttpServletRequest servletRequest, ArchiveHandler<O, E> handler)
TokenPrincipal tokenPrincipal, HttpServletRequest servletRequest, ArchiveHandler<O, E> handler)
throws IOException {
if (fileInfo.isDirectory()) {
handler.putNextEntry(supportDir, relPath);
......@@ -203,17 +191,8 @@ public class ArchiveService {
return;
}
if (fileInfo.getLocationId() != null && "portal".equals(fileInfo.getLocationType())) {
// remote file
if (portalLocationUrls == null) {
portalLocationUrls = locationDAO.getPortalLocationUrls();
}
String url = portalLocationUrls.get(fileInfo.getLocationId());
downloadRemoteLocationFileIntoArchive(fileInfo, relPath, tokenPrincipal, handler, url);
} else {
// local file or virtual directory
writeFileIntoArchive(fileInfo, relPath, tokenPrincipal, handler);
}
}
private File getArchiveFile(ArchiveJob job) throws IOException {
......@@ -387,21 +366,6 @@ public class ArchiveService {
}, new Object[]{});
}
private <O extends OutputStream, E> void downloadRemoteLocationFileIntoArchive(
FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,
ArchiveHandler<O, E> handler, String baseUrl) {
if (baseUrl == null) {
LOG.error("Location URL not found for location " + fileInfo.getLocationId());
throw new InternalFaultException("Unable to retrieve location of file "
+ fileInfo.getVirtualPath());
}
String url = baseUrl + "/" + fileInfo.getVirtualName();
downloadFromUrlIntoArchive(url, relPath, tokenPrincipal, handler);
}
private <O extends OutputStream, E> void downloadExternalLinkIntoArchive(
FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,
ArchiveHandler<O, E> handler, HttpServletRequest servletRequest) {
......
......@@ -7,23 +7,17 @@ package it.inaf.ia2.transfer.service;
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.FileDAO;
import it.inaf.ia2.transfer.persistence.LocationDAO;
import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.oats.vospace.exception.InternalFaultException;
import it.inaf.oats.vospace.exception.NodeNotFoundException;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
import it.inaf.oats.vospace.exception.QuotaExceededException;
import java.io.File;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
......@@ -35,9 +29,6 @@ public class FileCopyService {
@Autowired
private FileDAO fileDAO;
@Autowired
private LocationDAO locationDAO;
@Autowired
private AuthorizationService authorizationService;
......@@ -67,7 +58,6 @@ public class FileCopyService {
throw new NodeNotFoundException(sourceRootVosPath);
}
// Set location of destinations to this file service update location
// before retrieving file infos
fileDAO.setBranchLocationId(destinationRootVosPath, jobId, uploadLocationId);
......@@ -101,9 +91,6 @@ public class FileCopyService {
String destinationRootVosPath,
TokenPrincipal principal) {
// it will be initialized only when necessary
Map<Integer, String> portalLocationUrls = null;
for (FileInfo destinationFileInfo : destinationFileInfos) {
LOG.trace("Processing {} destination", destinationFileInfo.getVirtualPath());
// Cycle on files only
......@@ -134,24 +121,9 @@ public class FileCopyService {
}
}
if (sourceFileInfo.getLocationId() != null && "portal".equals(sourceFileInfo.getLocationType())) {
// remote file
if (portalLocationUrls == null) {
portalLocationUrls = locationDAO.getPortalLocationUrls();
}
String url = portalLocationUrls.get(sourceFileInfo.getLocationId());
// download file to destination disk path
this.downloadFileToDisk(sourceFileInfo,
destinationFileInfo,
principal, url, remainingQuota);
} else {
// local file
this.copyLocalFile(sourceFileInfo, destinationFileInfo, principal, remainingQuota);
}
}
}
}
......@@ -168,36 +140,6 @@ public class FileCopyService {
}
private void downloadFileToDisk(FileInfo sourceFileInfo,
FileInfo destinationFileInfo, TokenPrincipal tokenPrincipal, String baseUrl, Long remainingQuota) {
if (baseUrl == null) {
LOG.error("Location URL not found for location " + sourceFileInfo.getLocationId());
throw new InternalFaultException("Unable to retrieve location of file " + sourceFileInfo.getVirtualPath());
}
String url = baseUrl + "/" + sourceFileInfo.getVirtualName();
LOG.trace("Downloading file from {}", url);
restTemplate.execute(url, HttpMethod.GET, req -> {
HttpHeaders headers = req.getHeaders();
if (tokenPrincipal.getToken() != null) {
headers.setBearerAuth(tokenPrincipal.getToken());
}
}, res -> {
try (InputStream in = res.getBody()) {
putFileService.storeFileFromInputStream(sourceFileInfo, destinationFileInfo, in, remainingQuota);
} catch (Exception ex) {
// outFile.delete();
throw new RuntimeException(ex);
}
return null;
}, new Object[]{});
}
private void copyLocalFile(FileInfo sourceFileInfo,
FileInfo destinationFileInfo, TokenPrincipal tokenPrincipal, Long remainingQuota) {
......
/*
* This file is part of vospace-file-service
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.ia2.transfer.persistence;
import java.util.Map;
import javax.sql.DataSource;
import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
@ExtendWith(SpringExtension.class)
@ContextConfiguration(classes = {DataSourceConfig.class})
@TestPropertySource(locations = "classpath:test.properties")
public class LocationDAOTest {
@Autowired
private DataSource dataSource;
private LocationDAO dao;
@BeforeEach
public void init() {
dao = new LocationDAO(dataSource);
}
@Test
public void testGetPortalLocationUrls() {
Map<Integer, String> map = dao.getPortalLocationUrls();
assertEquals(1, map.size());
assertEquals("http://archive.lbto.org/files/lbt", map.get(4));
}
}
......@@ -8,7 +8,6 @@ package it.inaf.ia2.transfer.service;
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.FileDAO;
import it.inaf.ia2.transfer.persistence.JobDAO;
import it.inaf.ia2.transfer.persistence.LocationDAO;
import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.oats.vospace.exception.QuotaExceededException;
import it.inaf.oats.vospace.parent.exchange.ArchiveEntryDescriptor;
......@@ -49,7 +48,6 @@ import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.Primary;
import org.springframework.http.HttpMethod;
import org.springframework.http.client.ClientHttpResponse;
import org.springframework.test.context.ContextConfiguration;
......@@ -68,9 +66,6 @@ public class ArchiveServiceTest {
@MockBean
private FileDAO fileDAO;
@MockBean
private LocationDAO locationDAO;
@MockBean
private RestTemplate restTemplate;
......@@ -195,7 +190,6 @@ public class ArchiveServiceTest {
File file4 = createFile(tmpParent, "2021/10/1/UUID-file4");
File file5 = createFile(tmpParent, "2021/10/1/UUID-file5");
File file6 = createFile(tmpParent, "2021/10/1/UUID-file6");
File file7 = createFile(tmpParent, "2021/10/1/UUID-portal-file");
ArchiveJob job = new ArchiveJob();
job.setPrincipal(new TokenPrincipal("user1", "token1"));
......@@ -226,12 +220,10 @@ public class ArchiveServiceTest {
addFileInfo(fileInfos, parent + "/dir2/c/file4", file4);
addDirInfo(fileInfos, parent + "/dir2/c/d");
addFileInfo(fileInfos, parent + "/dir2/c/d/file5", file5);
addFileInfo(fileInfos, parent + "/portal-file", file7).setLocationId(1);
when(fileDAO.getArchiveFileInfos(any())).thenReturn(fileInfos);
when(locationDAO.getPortalLocationUrls()).thenReturn(Map.of(1, "http://portal/base/url"));
/*
doAnswer(invocation -> {
ResponseExtractor responseExtractor = invocation.getArgument(3);
ClientHttpResponse mockedResponse = mock(ClientHttpResponse.class);
......@@ -240,6 +232,7 @@ public class ArchiveServiceTest {
return null;
}).when(restTemplate).execute(eq("http://portal/base/url/portal-file"), eq(HttpMethod.GET),
any(RequestCallback.class), any(ResponseExtractor.class), any(Object[].class));
*/
archiveService.createArchive(job, servletRequest);
......@@ -250,7 +243,7 @@ public class ArchiveServiceTest {
// verify result structure
List<String> expectedSequence = Arrays.asList("file6", "dir1/", "dir1/a/", "dir1/a/b/",
"dir1/a/b/file1", "dir1/a/b/file2", "dir2/", "dir2/c/", "dir2/c/file3", "dir2/c/file4",
"dir2/c/d/", "dir2/c/d/file5", "portal-file");
"dir2/c/d/", "dir2/c/d/file5");
int i = 0;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment