Skip to content
Snippets Groups Projects
Commit 85b0e02d authored by Sonia Zorba's avatar Sonia Zorba
Browse files

Implemented async recall of multiple files

parent 17eb0e40
No related branches found
No related tags found
No related merge requests found
Pipeline #893 passed
......@@ -9,6 +9,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.web.servlet.FilterRegistrationBean;
import org.springframework.context.annotation.Bean;
import org.springframework.core.Ordered;
import org.springframework.web.client.RestTemplate;
@SpringBootApplication
public class VOSpaceUiApplication {
......@@ -47,4 +48,9 @@ public class VOSpaceUiApplication {
int parallelism = Math.min(0x7fff /* copied from ForkJoinPool.java */, Runtime.getRuntime().availableProcessors());
return new ForkJoinPool(parallelism, threadFactory, null, false);
}
@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
}
}
......@@ -2,23 +2,34 @@ package it.inaf.ia2.vospace.ui.controller;
import it.inaf.ia2.vospace.ui.client.VOSpaceClient;
import it.inaf.ia2.vospace.ui.data.Job;
import it.inaf.ia2.vospace.ui.exception.BadRequestException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import net.ivoa.xml.uws.v1.ExecutionPhase;
import net.ivoa.xml.uws.v1.JobSummary;
import net.ivoa.xml.vospace.v2.Protocol;
import net.ivoa.xml.vospace.v2.StructuredDataNode;
import net.ivoa.xml.vospace.v2.Transfer;
import net.ivoa.xml.vospace.v2.View;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestTemplate;
@RestController
public class JobController {
public class JobController extends BaseController {
@Value("${vospace-authority}")
private String authority;
......@@ -26,16 +37,26 @@ public class JobController {
@Autowired
private VOSpaceClient client;
@Autowired
private RestTemplate restTemplate;
@PostMapping(value = "/recall", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<Job> startRecallFromTapeJob(@RequestBody List<String> paths) {
if (paths.size() != 1) {
throw new UnsupportedOperationException();
if (paths.isEmpty()) {
throw new BadRequestException("Received empty list of nodes");
}
String target;
if (paths.size() == 1) {
target = "vos://" + authority + paths.get(0);
} else {
target = createTempListOfFilesNode(paths);
}
Transfer transfer = new Transfer();
transfer.setDirection("pullToVoSpace");
transfer.setTarget("vos://" + authority + paths.get(0));
transfer.setTarget(target);
Protocol protocol = new Protocol();
protocol.setUri("ia2:tape-recall");
transfer.getProtocols().add(protocol);
......@@ -49,6 +70,85 @@ public class JobController {
throw new RuntimeException("Error while executing job " + job.getJobId() + ". Job phase is " + job.getPhase() + ". QUEUED expected");
}
private String createTempListOfFilesNode(List<String> paths) {
StructuredDataNode dataNode = createStructuredDataNode(paths);
client.createNode(dataNode);
String uploadEndpoint = getTempFileEndpoint(dataNode.getUri());
String content = String.join("\n", paths);
upload(uploadEndpoint, content);
return dataNode.getUri();
}
private StructuredDataNode createStructuredDataNode(List<String> paths) {
List<View> views = new ArrayList<>();
View view = new View();
view.setUri("urn:list-of-files");
views.add(view);
StructuredDataNode dataNode = new StructuredDataNode();
String parentPath = getParentPath(paths);
String newTempFile = ".tmp-" + UUID.randomUUID().toString().replace("-", "") + ".txt";
dataNode.setUri("vos://" + authority + parentPath + "/" + newTempFile);
dataNode.setAccepts(views);
dataNode.setProvides(views);
return dataNode;
}
private String getParentPath(List<String> paths) {
// All the paths have the same parent, we can choose the first for extracting the path
String firstPath = paths.get(0);
return firstPath.substring(0, firstPath.lastIndexOf("/"));
}
private String getTempFileEndpoint(String target) {
Transfer transfer = new Transfer();
transfer.setDirection("pushToVoSpace");
transfer.setTarget(target);
Protocol protocol = new Protocol();
protocol.setUri("ivo://ivoa.net/vospace/core#httpget");
transfer.getProtocols().add(protocol);
return client.getFileServiceEndpoints(transfer).get(0).getEndpoint();
}
private void upload(String endpoint, String content) {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
MultiValueMap<String, Object> parts = new LinkedMultiValueMap<>();
parts.add("file", new MultipartFileResource(content, "list.txt"));
HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<>(parts, headers);
restTemplate.exchange(endpoint, HttpMethod.PUT, requestEntity, Void.class);
}
private class MultipartFileResource extends ByteArrayResource {
private final String fileName;
public MultipartFileResource(String content, String fileName) {
super(content.getBytes());
this.fileName = fileName;
}
@Override
public String getFilename() {
return this.fileName;
}
}
@GetMapping(value = "/jobs", produces = MediaType.APPLICATION_JSON_VALUE)
public List<Job> getJobs() {
// TODO
......
......@@ -48,6 +48,11 @@ public class NodesService {
NodeInfo nodeInfo = new NodeInfo(node, authority);
if (nodeInfo.getName().startsWith(".")) {
// hidden file
return "";
}
String html = "<tr>";
html += "<td><input type=\"checkbox\" data-node=\"" + nodeInfo.getPath() + "\" /></td>";
html += "<td>" + getIcon(nodeInfo) + getLink(nodeInfo) + "</td>";
......
package it.inaf.ia2.vospace.ui.controller;
import it.inaf.ia2.vospace.ui.client.VOSpaceClient;
import java.util.Collections;
import net.ivoa.xml.uws.v1.ExecutionPhase;
import net.ivoa.xml.uws.v1.JobSummary;
import net.ivoa.xml.vospace.v2.Protocol;
import org.junit.jupiter.api.Test;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.Mockito.when;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.http.MediaType;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.web.servlet.MockMvc;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import org.springframework.web.client.RestTemplate;
@SpringBootTest
@AutoConfigureMockMvc
@TestPropertySource(properties = {"vospace-authority=example.com!vospace"})
public class JobControllerTest {
@MockBean
private VOSpaceClient client;
@MockBean
private RestTemplate restTemplate;
@Autowired
private MockMvc mockMvc;
@Test
public void testSingleFileAsyncRecall() throws Exception {
JobSummary job = new JobSummary();
job.setPhase(ExecutionPhase.QUEUED);
when(client.startTransferJob(any())).thenReturn(job);
mockMvc.perform(post("/recall")
.contentType(MediaType.APPLICATION_JSON)
.content("[\"/path/to/file\"]"))
.andExpect(status().isOk());
}
@Test
public void testMultipleFilesAsyncRecall() throws Exception {
JobSummary job = new JobSummary();
job.setPhase(ExecutionPhase.QUEUED);
when(client.startTransferJob(argThat(transfer -> {
return transfer.getTarget().startsWith("vos://example.com!vospace/path/to/.tmp-");
}))).thenReturn(job);
Protocol protocol = new Protocol();
protocol.setEndpoint("http://file-service/path/to/file");
when(client.getFileServiceEndpoints(any())).thenReturn(Collections.singletonList(protocol));
mockMvc.perform(post("/recall")
.contentType(MediaType.APPLICATION_JSON)
.content("[\"/path/to/file1\", \"/path/to/file2\"]"))
.andExpect(status().isOk());
}
}
package it.inaf.ia2.vospace.ui.controller;
import it.inaf.ia2.vospace.ui.service.NodesService;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify;
......@@ -12,7 +11,6 @@ import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.web.servlet.MockMvc;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
@SpringBootTest
@AutoConfigureMockMvc
......@@ -21,17 +19,9 @@ public class NodesControllerTest {
@MockBean
private NodesService nodesService;
@Autowired
private NodesController controller;
@Autowired
private MockMvc mockMvc;
@BeforeEach
public void init() {
mockMvc = MockMvcBuilders.standaloneSetup(controller).build();
}
@Test
public void testListNodesEmpty() throws Exception {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment