From b4e11d2864d20e674a6ae7a99ec9270127691f3f Mon Sep 17 00:00:00 2001
From: Nicola Fulvio Calabria <nicola.calabria@inaf.it>
Date: Thu, 2 Dec 2021 12:56:38 +0100
Subject: [PATCH] Commented out or deleted code about portal locations
---
.../ia2/transfer/persistence/LocationDAO.java | 44 -------------
.../ia2/transfer/service/ArchiveService.java | 62 ++++---------------
.../ia2/transfer/service/FileCopyService.java | 62 +------------------
.../transfer/persistence/LocationDAOTest.java | 41 ------------
.../transfer/service/ArchiveServiceTest.java | 17 ++---
5 files changed, 20 insertions(+), 206 deletions(-)
delete mode 100644 src/main/java/it/inaf/ia2/transfer/persistence/LocationDAO.java
delete mode 100644 src/test/java/it/inaf/ia2/transfer/persistence/LocationDAOTest.java
diff --git a/src/main/java/it/inaf/ia2/transfer/persistence/LocationDAO.java b/src/main/java/it/inaf/ia2/transfer/persistence/LocationDAO.java
deleted file mode 100644
index a84ec7b..0000000
--- a/src/main/java/it/inaf/ia2/transfer/persistence/LocationDAO.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * This file is part of vospace-file-service
- * Copyright (C) 2021 Istituto Nazionale di Astrofisica
- * SPDX-License-Identifier: GPL-3.0-or-later
- */
-package it.inaf.ia2.transfer.persistence;
-
-import java.util.HashMap;
-import java.util.Map;
-import javax.sql.DataSource;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.jdbc.core.JdbcTemplate;
-import org.springframework.stereotype.Repository;
-
-@Repository
-public class LocationDAO {
-
- private final JdbcTemplate jdbcTemplate;
-
- @Autowired
- public LocationDAO(DataSource fileCatalogDatasource) {
- this.jdbcTemplate = new JdbcTemplate(fileCatalogDatasource);
- }
-
- public Map<Integer, String> getPortalLocationUrls() {
-
- String sql = "SELECT location_id, hostname, base_url\n"
- + "FROM location l\n"
- + "JOIN storage s ON s.storage_id = l.storage_dest_id\n"
- + "WHERE location_type = 'portal'";
-
- return jdbcTemplate.query(sql, rs -> {
- Map<Integer, String> locationUrls = new HashMap<>();
- while (rs.next()) {
- int locationId = rs.getInt("location_id");
- String hostname = rs.getString("hostname");
- String baseUrl = rs.getString("base_url");
- String url = "http://" + hostname + baseUrl;
- locationUrls.put(locationId, url);
- }
- return locationUrls;
- });
- }
-}
diff --git a/src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java b/src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java
index 3c9e767..93e6976 100644
--- a/src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java
+++ b/src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java
@@ -6,12 +6,10 @@
package it.inaf.ia2.transfer.service;
import it.inaf.ia2.aa.ServletRapClient;
-import it.inaf.ia2.aa.data.User;
import it.inaf.ia2.rap.client.call.TokenExchangeRequest;
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.FileDAO;
import it.inaf.ia2.transfer.persistence.JobDAO;
-import it.inaf.ia2.transfer.persistence.LocationDAO;
import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.oats.vospace.exception.InternalFaultException;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
@@ -57,9 +55,6 @@ public class ArchiveService {
@Autowired
private FileDAO fileDAO;
- @Autowired
- private LocationDAO locationDAO;
-
@Autowired
private LinkedServiceDAO linkedServiceDAO;
@@ -75,9 +70,6 @@ public class ArchiveService {
@Autowired
private ServletRapClient rapClient;
- @Value("${upload_location_id}")
- private int uploadLocationId;
-
// Directory containing temporary files generated by jobs.
@Value("${generated.dir}")
private String generatedDirString;
@@ -109,13 +101,11 @@ public class ArchiveService {
List<ArchiveEntryDescriptor> entryDescriptors = job.getEntryDescriptors();
- String commonParent = getCommonParent(entryDescriptors);
-
// support directory used to generate folder inside tar files (path is redefined each time by TarEntry class)
File supportDir = Files.createTempDirectory("dir").toFile();
- try ( ArchiveHandler<O, E> handler = getArchiveHandler(archiveFile, job.getType())) {
- fillArchive(entryDescriptors, commonParent, supportDir,
+ try (ArchiveHandler<O, E> handler = getArchiveHandler(archiveFile, job.getType())) {
+ fillArchive(entryDescriptors, supportDir,
job.getPrincipal(), servletRequest, handler);
} finally {
FileSystemUtils.deleteRecursively(supportDir);
@@ -127,13 +117,10 @@ public class ArchiveService {
}
private <O extends OutputStream, E> void fillArchive(
- List<ArchiveEntryDescriptor> entryDescriptors, String commonParent,
+ List<ArchiveEntryDescriptor> entryDescriptors,
File supportDir, TokenPrincipal tokenPrincipal,
HttpServletRequest servletRequest, ArchiveHandler<O, E> handler) throws IOException {
- // it will be initialized only when necessary
- Map<Integer, String> portalLocationUrls = null;
-
List<ArchiveEntryDescriptor> noTargetEntryDescriptors
= entryDescriptors.stream().filter(ed -> !ed.isPointingToAnotherNode())
.collect(Collectors.toList());
@@ -142,13 +129,15 @@ public class ArchiveService {
List<String> vosPaths = noTargetEntryDescriptors.stream()
.map(ed -> ed.getVosPath())
.collect(Collectors.toList());
+
+ String commonParent = getCommonParent(entryDescriptors);
if (!vosPaths.isEmpty()) {
for (FileInfo fileInfo : fileDAO.getArchiveFileInfos(vosPaths)) {
String relPath = fileInfo.getVirtualPath().substring(commonParent.length());
- this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, portalLocationUrls, servletRequest, handler);
+ this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, servletRequest, handler);
}
}
@@ -175,7 +164,7 @@ public class ArchiveService {
for (String vosPath : linkVosPaths) {
String relPath = vosPath.substring(commonParent.length());
- this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, portalLocationUrls, servletRequest, handler);
+ this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, servletRequest, handler);
}
}
}
@@ -183,8 +172,7 @@ public class ArchiveService {
private <O extends OutputStream, E> void insertEntryIntoArchive(
FileInfo fileInfo, File supportDir, String relPath,
- TokenPrincipal tokenPrincipal, Map<Integer, String> portalLocationUrls,
- HttpServletRequest servletRequest, ArchiveHandler<O, E> handler)
+ TokenPrincipal tokenPrincipal, HttpServletRequest servletRequest, ArchiveHandler<O, E> handler)
throws IOException {
if (fileInfo.isDirectory()) {
handler.putNextEntry(supportDir, relPath);
@@ -202,18 +190,9 @@ public class ArchiveService {
}
return;
}
+
+ writeFileIntoArchive(fileInfo, relPath, tokenPrincipal, handler);
- if (fileInfo.getLocationId() != null && "portal".equals(fileInfo.getLocationType())) {
- // remote file
- if (portalLocationUrls == null) {
- portalLocationUrls = locationDAO.getPortalLocationUrls();
- }
- String url = portalLocationUrls.get(fileInfo.getLocationId());
- downloadRemoteLocationFileIntoArchive(fileInfo, relPath, tokenPrincipal, handler, url);
- } else {
- // local file or virtual directory
- writeFileIntoArchive(fileInfo, relPath, tokenPrincipal, handler);
- }
}
private File getArchiveFile(ArchiveJob job) throws IOException {
@@ -374,10 +353,10 @@ public class ArchiveService {
}
}, res -> {
File tmpFile = Files.createTempFile("download", null).toFile();
- try ( FileOutputStream os = new FileOutputStream(tmpFile)) {
+ try (FileOutputStream os = new FileOutputStream(tmpFile)) {
res.getBody().transferTo(os);
handler.putNextEntry(tmpFile, relPath);
- try ( FileInputStream is = new FileInputStream(tmpFile)) {
+ try (FileInputStream is = new FileInputStream(tmpFile)) {
is.transferTo(handler.getOutputStream());
}
} finally {
@@ -387,21 +366,6 @@ public class ArchiveService {
}, new Object[]{});
}
- private <O extends OutputStream, E> void downloadRemoteLocationFileIntoArchive(
- FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,
- ArchiveHandler<O, E> handler, String baseUrl) {
-
- if (baseUrl == null) {
- LOG.error("Location URL not found for location " + fileInfo.getLocationId());
- throw new InternalFaultException("Unable to retrieve location of file "
- + fileInfo.getVirtualPath());
- }
-
- String url = baseUrl + "/" + fileInfo.getVirtualName();
-
- downloadFromUrlIntoArchive(url, relPath, tokenPrincipal, handler);
- }
-
private <O extends OutputStream, E> void downloadExternalLinkIntoArchive(
FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,
ArchiveHandler<O, E> handler, HttpServletRequest servletRequest) {
@@ -431,7 +395,7 @@ public class ArchiveService {
File file = new File(fileInfo.getFilePath());
LOG.trace("Adding file " + file.getAbsolutePath() + " to tar archive");
- try ( InputStream is = new FileInputStream(file)) {
+ try (InputStream is = new FileInputStream(file)) {
handler.putNextEntry(file, relPath);
is.transferTo(handler.getOutputStream());
}
diff --git a/src/main/java/it/inaf/ia2/transfer/service/FileCopyService.java b/src/main/java/it/inaf/ia2/transfer/service/FileCopyService.java
index 73634e5..ec53fce 100644
--- a/src/main/java/it/inaf/ia2/transfer/service/FileCopyService.java
+++ b/src/main/java/it/inaf/ia2/transfer/service/FileCopyService.java
@@ -7,23 +7,17 @@ package it.inaf.ia2.transfer.service;
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.FileDAO;
-import it.inaf.ia2.transfer.persistence.LocationDAO;
import it.inaf.ia2.transfer.persistence.model.FileInfo;
-import it.inaf.oats.vospace.exception.InternalFaultException;
import it.inaf.oats.vospace.exception.NodeNotFoundException;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
import it.inaf.oats.vospace.exception.QuotaExceededException;
import java.io.File;
-import java.io.InputStream;
import java.util.List;
-import java.util.Map;
import java.util.Optional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
@@ -35,9 +29,6 @@ public class FileCopyService {
@Autowired
private FileDAO fileDAO;
- @Autowired
- private LocationDAO locationDAO;
-
@Autowired
private AuthorizationService authorizationService;
@@ -67,7 +58,6 @@ public class FileCopyService {
throw new NodeNotFoundException(sourceRootVosPath);
}
-
// Set location of destinations to this file service update location
// before retrieving file infos
fileDAO.setBranchLocationId(destinationRootVosPath, jobId, uploadLocationId);
@@ -101,9 +91,6 @@ public class FileCopyService {
String destinationRootVosPath,
TokenPrincipal principal) {
- // it will be initialized only when necessary
- Map<Integer, String> portalLocationUrls = null;
-
for (FileInfo destinationFileInfo : destinationFileInfos) {
LOG.trace("Processing {} destination", destinationFileInfo.getVirtualPath());
// Cycle on files only
@@ -134,22 +121,7 @@ public class FileCopyService {
}
}
- if (sourceFileInfo.getLocationId() != null && "portal".equals(sourceFileInfo.getLocationType())) {
- // remote file
- if (portalLocationUrls == null) {
- portalLocationUrls = locationDAO.getPortalLocationUrls();
- }
- String url = portalLocationUrls.get(sourceFileInfo.getLocationId());
- // download file to destination disk path
- this.downloadFileToDisk(sourceFileInfo,
- destinationFileInfo,
- principal, url, remainingQuota);
-
- } else {
- // local file
- this.copyLocalFile(sourceFileInfo, destinationFileInfo, principal, remainingQuota);
-
- }
+ this.copyLocalFile(sourceFileInfo, destinationFileInfo, principal, remainingQuota);
}
}
@@ -168,36 +140,6 @@ public class FileCopyService {
}
- private void downloadFileToDisk(FileInfo sourceFileInfo,
- FileInfo destinationFileInfo, TokenPrincipal tokenPrincipal, String baseUrl, Long remainingQuota) {
-
- if (baseUrl == null) {
- LOG.error("Location URL not found for location " + sourceFileInfo.getLocationId());
- throw new InternalFaultException("Unable to retrieve location of file " + sourceFileInfo.getVirtualPath());
- }
-
- String url = baseUrl + "/" + sourceFileInfo.getVirtualName();
-
- LOG.trace("Downloading file from {}", url);
-
- restTemplate.execute(url, HttpMethod.GET, req -> {
- HttpHeaders headers = req.getHeaders();
- if (tokenPrincipal.getToken() != null) {
- headers.setBearerAuth(tokenPrincipal.getToken());
- }
- }, res -> {
-
- try (InputStream in = res.getBody()) {
- putFileService.storeFileFromInputStream(sourceFileInfo, destinationFileInfo, in, remainingQuota);
- } catch (Exception ex) {
- // outFile.delete();
- throw new RuntimeException(ex);
- }
-
- return null;
- }, new Object[]{});
- }
-
private void copyLocalFile(FileInfo sourceFileInfo,
FileInfo destinationFileInfo, TokenPrincipal tokenPrincipal, Long remainingQuota) {
@@ -207,7 +149,7 @@ public class FileCopyService {
}
File file = new File(sourceFileInfo.getFilePath());
- LOG.trace("Copying file: {} to {}",file.getAbsolutePath(), destinationFileInfo.getFilePath());
+ LOG.trace("Copying file: {} to {}", file.getAbsolutePath(), destinationFileInfo.getFilePath());
putFileService.copyLocalFile(sourceFileInfo, destinationFileInfo, remainingQuota);
diff --git a/src/test/java/it/inaf/ia2/transfer/persistence/LocationDAOTest.java b/src/test/java/it/inaf/ia2/transfer/persistence/LocationDAOTest.java
deleted file mode 100644
index 9e302eb..0000000
--- a/src/test/java/it/inaf/ia2/transfer/persistence/LocationDAOTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * This file is part of vospace-file-service
- * Copyright (C) 2021 Istituto Nazionale di Astrofisica
- * SPDX-License-Identifier: GPL-3.0-or-later
- */
-package it.inaf.ia2.transfer.persistence;
-
-import java.util.Map;
-import javax.sql.DataSource;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.TestPropertySource;
-import org.springframework.test.context.junit.jupiter.SpringExtension;
-
-@ExtendWith(SpringExtension.class)
-@ContextConfiguration(classes = {DataSourceConfig.class})
-@TestPropertySource(locations = "classpath:test.properties")
-public class LocationDAOTest {
-
- @Autowired
- private DataSource dataSource;
-
- private LocationDAO dao;
-
- @BeforeEach
- public void init() {
- dao = new LocationDAO(dataSource);
- }
-
- @Test
- public void testGetPortalLocationUrls() {
- Map<Integer, String> map = dao.getPortalLocationUrls();
-
- assertEquals(1, map.size());
- assertEquals("http://archive.lbto.org/files/lbt", map.get(4));
- }
-}
diff --git a/src/test/java/it/inaf/ia2/transfer/service/ArchiveServiceTest.java b/src/test/java/it/inaf/ia2/transfer/service/ArchiveServiceTest.java
index f145c45..0c86007 100644
--- a/src/test/java/it/inaf/ia2/transfer/service/ArchiveServiceTest.java
+++ b/src/test/java/it/inaf/ia2/transfer/service/ArchiveServiceTest.java
@@ -8,7 +8,6 @@ package it.inaf.ia2.transfer.service;
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.FileDAO;
import it.inaf.ia2.transfer.persistence.JobDAO;
-import it.inaf.ia2.transfer.persistence.LocationDAO;
import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.oats.vospace.exception.QuotaExceededException;
import it.inaf.oats.vospace.parent.exchange.ArchiveEntryDescriptor;
@@ -49,7 +48,6 @@ import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
-import org.springframework.context.annotation.Primary;
import org.springframework.http.HttpMethod;
import org.springframework.http.client.ClientHttpResponse;
import org.springframework.test.context.ContextConfiguration;
@@ -68,9 +66,6 @@ public class ArchiveServiceTest {
@MockBean
private FileDAO fileDAO;
- @MockBean
- private LocationDAO locationDAO;
-
@MockBean
private RestTemplate restTemplate;
@@ -195,7 +190,6 @@ public class ArchiveServiceTest {
File file4 = createFile(tmpParent, "2021/10/1/UUID-file4");
File file5 = createFile(tmpParent, "2021/10/1/UUID-file5");
File file6 = createFile(tmpParent, "2021/10/1/UUID-file6");
- File file7 = createFile(tmpParent, "2021/10/1/UUID-portal-file");
ArchiveJob job = new ArchiveJob();
job.setPrincipal(new TokenPrincipal("user1", "token1"));
@@ -225,13 +219,11 @@ public class ArchiveServiceTest {
addFileInfo(fileInfos, parent + "/dir2/c/file3", file3);
addFileInfo(fileInfos, parent + "/dir2/c/file4", file4);
addDirInfo(fileInfos, parent + "/dir2/c/d");
- addFileInfo(fileInfos, parent + "/dir2/c/d/file5", file5);
- addFileInfo(fileInfos, parent + "/portal-file", file7).setLocationId(1);
+ addFileInfo(fileInfos, parent + "/dir2/c/d/file5", file5);
when(fileDAO.getArchiveFileInfos(any())).thenReturn(fileInfos);
-
- when(locationDAO.getPortalLocationUrls()).thenReturn(Map.of(1, "http://portal/base/url"));
-
+
+ /*
doAnswer(invocation -> {
ResponseExtractor responseExtractor = invocation.getArgument(3);
ClientHttpResponse mockedResponse = mock(ClientHttpResponse.class);
@@ -240,6 +232,7 @@ public class ArchiveServiceTest {
return null;
}).when(restTemplate).execute(eq("http://portal/base/url/portal-file"), eq(HttpMethod.GET),
any(RequestCallback.class), any(ResponseExtractor.class), any(Object[].class));
+ */
archiveService.createArchive(job, servletRequest);
@@ -250,7 +243,7 @@ public class ArchiveServiceTest {
// verify result structure
List<String> expectedSequence = Arrays.asList("file6", "dir1/", "dir1/a/", "dir1/a/b/",
"dir1/a/b/file1", "dir1/a/b/file2", "dir2/", "dir2/c/", "dir2/c/file3", "dir2/c/file4",
- "dir2/c/d/", "dir2/c/d/file5", "portal-file");
+ "dir2/c/d/", "dir2/c/d/file5");
int i = 0;
--
GitLab