From 85b0e02ddaf5793ff0442795f2ad32e8b7370e78 Mon Sep 17 00:00:00 2001 From: Sonia Zorba Date: Wed, 20 Jan 2021 17:50:35 +0100 Subject: [PATCH] Implemented async recall of multiple files --- .../ia2/vospace/ui/VOSpaceUiApplication.java | 6 + .../vospace/ui/controller/JobController.java | 108 +++++++++++++++++- .../ia2/vospace/ui/service/NodesService.java | 5 + .../ui/controller/JobControllerTest.java | 71 ++++++++++++ .../ui/controller/NodesControllerTest.java | 10 -- 5 files changed, 186 insertions(+), 14 deletions(-) create mode 100644 vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/JobControllerTest.java diff --git a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/VOSpaceUiApplication.java b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/VOSpaceUiApplication.java index 461d692..a015518 100644 --- a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/VOSpaceUiApplication.java +++ b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/VOSpaceUiApplication.java @@ -9,6 +9,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.context.annotation.Bean; import org.springframework.core.Ordered; +import org.springframework.web.client.RestTemplate; @SpringBootApplication public class VOSpaceUiApplication { @@ -47,4 +48,9 @@ public class VOSpaceUiApplication { int parallelism = Math.min(0x7fff /* copied from ForkJoinPool.java */, Runtime.getRuntime().availableProcessors()); return new ForkJoinPool(parallelism, threadFactory, null, false); } + + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } } diff --git a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/JobController.java b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/JobController.java index 904a26b..1dd0638 100644 --- a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/JobController.java +++ b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/JobController.java @@ -2,23 +2,34 @@ package it.inaf.ia2.vospace.ui.controller; import it.inaf.ia2.vospace.ui.client.VOSpaceClient; import it.inaf.ia2.vospace.ui.data.Job; +import it.inaf.ia2.vospace.ui.exception.BadRequestException; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import net.ivoa.xml.uws.v1.ExecutionPhase; import net.ivoa.xml.uws.v1.JobSummary; import net.ivoa.xml.vospace.v2.Protocol; +import net.ivoa.xml.vospace.v2.StructuredDataNode; import net.ivoa.xml.vospace.v2.Transfer; +import net.ivoa.xml.vospace.v2.View; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.client.RestTemplate; @RestController -public class JobController { +public class JobController extends BaseController { @Value("${vospace-authority}") private String authority; @@ -26,16 +37,26 @@ public class JobController { @Autowired private VOSpaceClient client; + @Autowired + private RestTemplate restTemplate; + @PostMapping(value = "/recall", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity startRecallFromTapeJob(@RequestBody List paths) { - if (paths.size() != 1) { - throw new UnsupportedOperationException(); + if (paths.isEmpty()) { + throw new BadRequestException("Received empty list of nodes"); + } + + String target; + if (paths.size() == 1) { + target = "vos://" + authority + paths.get(0); + } else { + target = createTempListOfFilesNode(paths); } Transfer transfer = new Transfer(); transfer.setDirection("pullToVoSpace"); - transfer.setTarget("vos://" + authority + paths.get(0)); + transfer.setTarget(target); Protocol protocol = new Protocol(); protocol.setUri("ia2:tape-recall"); transfer.getProtocols().add(protocol); @@ -49,6 +70,85 @@ public class JobController { throw new RuntimeException("Error while executing job " + job.getJobId() + ". Job phase is " + job.getPhase() + ". QUEUED expected"); } + private String createTempListOfFilesNode(List paths) { + + StructuredDataNode dataNode = createStructuredDataNode(paths); + client.createNode(dataNode); + String uploadEndpoint = getTempFileEndpoint(dataNode.getUri()); + + String content = String.join("\n", paths); + + upload(uploadEndpoint, content); + + return dataNode.getUri(); + } + + private StructuredDataNode createStructuredDataNode(List paths) { + + List views = new ArrayList<>(); + View view = new View(); + view.setUri("urn:list-of-files"); + views.add(view); + + StructuredDataNode dataNode = new StructuredDataNode(); + + String parentPath = getParentPath(paths); + String newTempFile = ".tmp-" + UUID.randomUUID().toString().replace("-", "") + ".txt"; + + dataNode.setUri("vos://" + authority + parentPath + "/" + newTempFile); + + dataNode.setAccepts(views); + dataNode.setProvides(views); + + return dataNode; + } + + private String getParentPath(List paths) { + // All the paths have the same parent, we can choose the first for extracting the path + String firstPath = paths.get(0); + return firstPath.substring(0, firstPath.lastIndexOf("/")); + } + + private String getTempFileEndpoint(String target) { + + Transfer transfer = new Transfer(); + transfer.setDirection("pushToVoSpace"); + transfer.setTarget(target); + Protocol protocol = new Protocol(); + protocol.setUri("ivo://ivoa.net/vospace/core#httpget"); + transfer.getProtocols().add(protocol); + + return client.getFileServiceEndpoints(transfer).get(0).getEndpoint(); + } + + private void upload(String endpoint, String content) { + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + + MultiValueMap parts = new LinkedMultiValueMap<>(); + parts.add("file", new MultipartFileResource(content, "list.txt")); + + HttpEntity> requestEntity = new HttpEntity<>(parts, headers); + + restTemplate.exchange(endpoint, HttpMethod.PUT, requestEntity, Void.class); + } + + private class MultipartFileResource extends ByteArrayResource { + + private final String fileName; + + public MultipartFileResource(String content, String fileName) { + super(content.getBytes()); + this.fileName = fileName; + } + + @Override + public String getFilename() { + return this.fileName; + } + } + @GetMapping(value = "/jobs", produces = MediaType.APPLICATION_JSON_VALUE) public List getJobs() { // TODO diff --git a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/service/NodesService.java b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/service/NodesService.java index 91b0b82..8f5e6c6 100644 --- a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/service/NodesService.java +++ b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/service/NodesService.java @@ -48,6 +48,11 @@ public class NodesService { NodeInfo nodeInfo = new NodeInfo(node, authority); + if (nodeInfo.getName().startsWith(".")) { + // hidden file + return ""; + } + String html = ""; html += ""; html += "" + getIcon(nodeInfo) + getLink(nodeInfo) + ""; diff --git a/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/JobControllerTest.java b/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/JobControllerTest.java new file mode 100644 index 0000000..53128e3 --- /dev/null +++ b/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/JobControllerTest.java @@ -0,0 +1,71 @@ +package it.inaf.ia2.vospace.ui.controller; + +import it.inaf.ia2.vospace.ui.client.VOSpaceClient; +import java.util.Collections; +import net.ivoa.xml.uws.v1.ExecutionPhase; +import net.ivoa.xml.uws.v1.JobSummary; +import net.ivoa.xml.vospace.v2.Protocol; +import org.junit.jupiter.api.Test; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.when; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.web.servlet.MockMvc; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import org.springframework.web.client.RestTemplate; + +@SpringBootTest +@AutoConfigureMockMvc +@TestPropertySource(properties = {"vospace-authority=example.com!vospace"}) +public class JobControllerTest { + + @MockBean + private VOSpaceClient client; + + @MockBean + private RestTemplate restTemplate; + + @Autowired + private MockMvc mockMvc; + + @Test + public void testSingleFileAsyncRecall() throws Exception { + + JobSummary job = new JobSummary(); + job.setPhase(ExecutionPhase.QUEUED); + + when(client.startTransferJob(any())).thenReturn(job); + + mockMvc.perform(post("/recall") + .contentType(MediaType.APPLICATION_JSON) + .content("[\"/path/to/file\"]")) + .andExpect(status().isOk()); + } + + @Test + public void testMultipleFilesAsyncRecall() throws Exception { + + JobSummary job = new JobSummary(); + job.setPhase(ExecutionPhase.QUEUED); + + when(client.startTransferJob(argThat(transfer -> { + return transfer.getTarget().startsWith("vos://example.com!vospace/path/to/.tmp-"); + }))).thenReturn(job); + + Protocol protocol = new Protocol(); + protocol.setEndpoint("http://file-service/path/to/file"); + + when(client.getFileServiceEndpoints(any())).thenReturn(Collections.singletonList(protocol)); + + mockMvc.perform(post("/recall") + .contentType(MediaType.APPLICATION_JSON) + .content("[\"/path/to/file1\", \"/path/to/file2\"]")) + .andExpect(status().isOk()); + } +} diff --git a/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/NodesControllerTest.java b/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/NodesControllerTest.java index 9bd05f7..d4291fb 100644 --- a/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/NodesControllerTest.java +++ b/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/NodesControllerTest.java @@ -1,7 +1,6 @@ package it.inaf.ia2.vospace.ui.controller; import it.inaf.ia2.vospace.ui.service.NodesService; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; @@ -12,7 +11,6 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.web.servlet.MockMvc; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; @SpringBootTest @AutoConfigureMockMvc @@ -21,17 +19,9 @@ public class NodesControllerTest { @MockBean private NodesService nodesService; - @Autowired - private NodesController controller; - @Autowired private MockMvc mockMvc; - @BeforeEach - public void init() { - mockMvc = MockMvcBuilders.standaloneSetup(controller).build(); - } - @Test public void testListNodesEmpty() throws Exception { -- GitLab