diff --git a/transfer_service/preprocessor.py b/transfer_service/preprocessor.py index ffae00527058fbc366b1cbaaacbec3c3bb505254..9715af2b260f0af67272dd2b631c2b385ea50212 100644 --- a/transfer_service/preprocessor.py +++ b/transfer_service/preprocessor.py @@ -1,3 +1,5 @@ +import time + from multiprocessing import Process from config import Config @@ -11,9 +13,13 @@ class Preprocessor(Process): self.params = config.loadSection("scheduling") self.maxPendingJobs = self.params.getint("max_pending_jobs") self.maxReadyJobs = self.params.getint("max_ready_jobs") + self.execWaitTime = self.params.getint("exec_wait_time") self.pendingQueue = JobQueue("pending") self.readyQueue = JobQueue("ready") super(Preprocessor, self).__init__() + + def wait(self): + time.sleep(self.execWaitTime) def run(self): """ diff --git a/transfer_service/store_preprocessor.py b/transfer_service/store_preprocessor.py index aa27be0fd13f8dd9c3e5379156b726c0291e7e11..9c6c20f63fdf65db8b8c0a29dad31ae0aa5032ea 100644 --- a/transfer_service/store_preprocessor.py +++ b/transfer_service/store_preprocessor.py @@ -7,9 +7,9 @@ import json import os import shutil import sys -import time from datetime import datetime as dt + from checksum import Checksum from file_grouper import FileGrouper from db_connector import DbConnector @@ -169,7 +169,7 @@ class StorePreprocessor(Preprocessor): def run(self): while True: - time.sleep(10) + self.wait() if(self.readyQueue.len() <= self.maxReadyJobs and self.pendingQueue.len() > 0): self.jobObj = self.pendingQueue.getJob() self.username = self.jobObj["jobInfo"]["userName"] @@ -177,7 +177,6 @@ class StorePreprocessor(Preprocessor): self.execute() self.pendingQueue.moveJobTo("ready") print("Job MOVED!") - time.sleep(10) # Test #sp = StorePreprocessor() diff --git a/transfer_service/vos_ts.conf b/transfer_service/vos_ts.conf index d11956d20b06324547801a872c297a79bf2ee6b9..c2f99d5be3f182abcb3593e54faf9d326d26e1a8 100644 --- a/transfer_service/vos_ts.conf +++ b/transfer_service/vos_ts.conf @@ -43,7 +43,7 @@ port = 5672 [checksum] ; suffix for files containing MD5 checksums md5_file_suffix = -md5sum.txt -; buffer size in bytes when reading a chunk, default is 4096 B +; buffer size in bytes when reading a chunk of data, default is 4096 B file_buffer_size = 4096 [file_grouper] @@ -57,6 +57,9 @@ max_dir_size = 100 max_pending_jobs = 32 ; maximum number of jobs within a 'ready' queue, default is 4 max_ready_jobs = 4 +; minimum time interval in seconds between two consecutive checks on +; job queues (do NOT set to zero), default is 15 s +exec_wait_time = 15 ####################