Changeset f2616a3 in flex_extract.git

Timestamp:

Dec 14, 2018, 1:03:35 PM (5 years ago)

Author:

Anne Philipp <anne.philipp@…>

Branches:

master, ctbto, dev

Children:

0629ba8

Parents:

1eca806

Message:

implemented a job split with a new parameter 'job_chunk' so that huge time periods can automatically be splitted

Location:

source/python

Files:

• classes/ControlFile.py (modified) (2 diffs)
• mods/checks.py (modified) (1 diff)
• mods/tools.py (modified) (1 diff)
• submit.py (modified) (4 diffs)

source/python/classes/ControlFile.py

@@ -68 +68 @@
                          check_acctime, check_accmaxstep, check_time,
                          check_logicals_type, check_len_type_time_step,
-                         check_addpar)
+                         check_addpar, check_job_chunk)
 
 # ------------------------------------------------------------------------------
@@ -383 +383 @@
         self.addpar = check_addpar(self.addpar)
 
+        self.job_chunk = check_job_chunk(self.job_chunk)
 
         return

source/python/mods/checks.py

@@ -796 +796 @@
     return addpar
 
+
+def check_job_chunk(job_chunk):
+    '''Checks that the job chunk number is positive and non zero.
+
+    Parameters
+    ----------
+    job_chunk : :obj:`integer`
+        The number of days for a single job script.
+
+    Return
+    ------
+    job_chunk : :obj:`integer`
+        The number of days for a single job script.
+    '''
+    if job_chunk < 0:
+        raise ValueError('ERROR: The number of job chunk is negative!\n'
+                         'It has to be a positive number!')
+    elif job_chunk == 0:
+        job_chunk = None
+    else:
+        pass
+
+    return job_chunk

source/python/mods/tools.py

@@ -128 +128 @@
                         type=none_or_int, default=None,
                         help="# of days to be retrieved at once")
+    parser.add_argument("--job_chunk", dest="job_chunk",
+                        type=none_or_int, default=None,
+                        help="# of days to be retrieved within a single job")
     parser.add_argument("--controlfile", dest="controlfile",
                         type=none_or_str, default='CONTROL.temp',

source/python/submit.py

@@ -48 +48 @@
 import inspect
 import collections
+from datetime import datetime, timedelta
 
 # software specific classes and modules from flex_extract
@@ -140 +141 @@
                                 jtemplate[:-5] + '.ksh')
 
-        clist = c.to_list()
-
-        mk_jobscript(jtemplate, job_file, clist)
+        # divide time periode into specified number of job chunks
+        # to have multiple job scripts
+        if c.job_chunk:
+            start = datetime.strptime(c.start_date, '%Y%m%d')
+            end = datetime.strptime(c.end_date, '%Y%m%d')
+            chunk = timedelta(days=c.job_chunk)
+
+            while start <= end:
+                if (start + chunk) <= end:
+                    c.end_date = (start + chunk).strftime("%Y%m%d")
+                else:
+                    c.end_date = end.strftime("%Y%m%d")
+                print c.start_date +' bis ' + c.end_date
+
+                clist = c.to_list()
+
+                mk_jobscript(jtemplate, job_file, clist)
+
+                job_id = submit_job_to_ecserver(queue, job_file)
+                print('The job id is: ' + str(job_id.strip()))
+
+                start = start + chunk
+                c.start_date = start.strftime("%Y%m%d")
+        # submit a single job script
+        else:
+            clist = c.to_list()
+
+            mk_jobscript(jtemplate, job_file, clist)
+
+            job_id = submit_job_to_ecserver(queue, job_file)
+            print('The job id is: ' + str(job_id.strip()))
 
     else:
@@ -161 +190 @@
         mk_jobscript(jtemplate, job_file, clist)
 
-    # --------- submit the job_script to the ECMWF server
-    job_id = submit_job_to_ecserver(queue, job_file)
-    print('The job id is: ' + str(job_id.strip()))
-    print('You should get an email with subject flex.hostname.pid')
+        job_id = submit_job_to_ecserver(queue, job_file)
+        print('The job id is: ' + str(job_id.strip()))
+
+
+    print('You should get an email per job with subject flex.hostname.pid')
 
     return
@@ -223 +253 @@
     return
 
+
 if __name__ == "__main__":
     main()