Avoided OutOfMemoryError in MD5 calculation of big files

src/main/java/it/inaf/ia2/transfer/service/PutFileService.java

@@ -12,7 +12,10 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
 import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.Objects;
@@ -126,16 +129,16 @@ public class PutFileService {
                 LOG.warn("Destination file {} size mismatch with source", destinationFile.toPath().toString());
             }
 
-            if (sourceFileInfo.getContentType() != null && 
-                    !sourceFileInfo.getContentType().equals(destinationFileInfo.getContentType())) {
+            if (sourceFileInfo.getContentType() != null
+                    && !sourceFileInfo.getContentType().equals(destinationFileInfo.getContentType())) {
                 LOG.warn("Destination file {} content type mismatch with source {} {}", destinationFile.toPath().toString(),
                         destinationFileInfo.getContentType(), sourceFileInfo.getContentType());
             }
 
-            if (sourceFileInfo.getContentMd5() != null && 
-                    !sourceFileInfo.getContentMd5().equals(md5Checksum)) {
+            if (sourceFileInfo.getContentMd5() != null
+                    && !sourceFileInfo.getContentMd5().equals(md5Checksum)) {
                 LOG.warn("Destination file {} md5 mismatch with source {} {}", destinationFile.toPath().toString(),
-                        destinationFileInfo.getContentMd5(), sourceFileInfo.getContentMd5() );
+                        destinationFileInfo.getContentMd5(), sourceFileInfo.getContentMd5());
             }
 
         }
@@ -189,7 +192,27 @@ public class PutFileService {
 
     private String makeMD5Checksum(File file) throws NoSuchAlgorithmException, IOException {
         MessageDigest md = MessageDigest.getInstance("MD5");
-        md.update(Files.readAllBytes(file.toPath()));
+
+        // We can't update the MessageDigest object in a single step using
+        // Files.readAllBytes because we want to handle also big files and that
+        // method loads all the content in memory (OutOfMemoryError has been
+        // noticed for files bigger than 1GB). So multiple updates using
+        // FileChannel and ByteBuffer are used
+        try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) {
+            // creating buffer (by default it is in write mode). FileChannel will write into buffer
+            ByteBuffer buffer = ByteBuffer.allocate(1024);
+            int bytesRead;
+            while ((bytesRead = channel.read(buffer)) != -1) {
+                if (bytesRead > 0) {
+                    // switch from write to read mode
+                    buffer.flip();
+                    md.update(buffer);
+                }
+                // using rewind the buffer can be filled again
+                buffer.rewind();
+            }
+        }
+
         byte[] digest = md.digest();
         String checksum = DatatypeConverter.printHexBinary(digest);
         return checksum;

src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java

@@ -25,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.eq;
 import org.mockito.Mockito;
 import static org.mockito.Mockito.mock;
@@ -77,7 +78,7 @@ public class PutFileControllerTest {
     public void putGenericFile() throws Exception {
 
         when(fileDao.getRemainingQuota(any())).thenReturn(null);
-        
+
         String randomFileName = UUID.randomUUID().toString();
         createBaseFileInfo(randomFileName);
 
@@ -92,9 +93,12 @@ public class PutFileControllerTest {
 
         assertTrue(file.exists());
         assertEquals("content", Files.contentOf(file, StandardCharsets.UTF_8));
+
+        verify(fileDao, times(1)).updateFileAttributes(anyInt(), eq("text/plain"), any(), eq(7l), eq("9A0364B9E99BB480DD25E1F0284C8555"));
+
         assertTrue(file.delete());
     }
-    
+
     @Test
     public void putGenericFileWithNameConflictExtension() throws Exception {
         putGenericFileWithNameConflict("test.txt", "test-1.txt", "test-2.txt");
@@ -108,7 +112,7 @@ public class PutFileControllerTest {
     private void putGenericFileWithNameConflict(String name1, String name2, String name3) throws Exception {
 
         when(fileDao.getRemainingQuota(any())).thenReturn(null);
-        
+
         createBaseFileInfo(name1);
 
         MockMultipartFile fakeFile = new MockMultipartFile("file", "test.txt", "text/plain", "content".getBytes());
@@ -154,7 +158,7 @@ public class PutFileControllerTest {
     public void putGenericFileWithJobId() throws Exception {
 
         when(fileDao.getRemainingQuota(any())).thenReturn(null);
-        
+
         when(jobDAO.isJobExisting("pippo10")).thenReturn(false);
         when(jobDAO.isJobExisting("pippo5")).thenReturn(true);
 
@@ -171,7 +175,7 @@ public class PutFileControllerTest {
 
         verify(jobDAO, times(1)).isJobExisting(eq("pippo10"));
 
-        // Retry with valid jobid        
+        // Retry with valid jobid
         mockMvc.perform(putMultipart("/path/to/test.txt")
                 .file(fakeFile).param("jobId", "pippo5"))
                 .andDo(print())
@@ -233,7 +237,7 @@ public class PutFileControllerTest {
 
         verify(jobDAO, times(1)).setJobError(eq("abcdef"), any());
     }
-    
+
     @Test
     public void testQuotaExceededMultipart() throws Exception {
 
@@ -291,16 +295,16 @@ public class PutFileControllerTest {
         String randomFileName = UUID.randomUUID().toString();
         return createBaseFileInfo(randomFileName);
     }
-    
-    private FileInfo createBaseFileInfo(String fileName) {        
+
+    private FileInfo createBaseFileInfo(String fileName) {
         FileInfo fileInfo = new FileInfo();
         fileInfo.setOsPath(getTestFilePath(fileName));
         fileInfo.setVirtualPath("/path/to/" + fileName);
         fileInfo.setPublic(false);
-        
+
         when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo));
         when(fileDao.setBusy(any(), any())).thenReturn(1);
-        
+
         return fileInfo;
     }