Changeset f2616a3 in flex_extract.git for source/python/submit.py

Timestamp:

Dec 14, 2018, 1:03:35 PM (5 years ago)

Author:

Anne Philipp <anne.philipp@…>

Branches:

master, ctbto, dev

Children:

0629ba8

Parents:

1eca806

Message:

implemented a job split with a new parameter 'job_chunk' so that huge time periods can automatically be splitted

File:

• source/python/submit.py (modified) (4 diffs)

source/python/submit.py

@@ -48 +48 @@
 import inspect
 import collections
+from datetime import datetime, timedelta
 
 # software specific classes and modules from flex_extract
@@ -140 +141 @@
                                 jtemplate[:-5] + '.ksh')
 
-        clist = c.to_list()
-
-        mk_jobscript(jtemplate, job_file, clist)
+        # divide time periode into specified number of job chunks
+        # to have multiple job scripts
+        if c.job_chunk:
+            start = datetime.strptime(c.start_date, '%Y%m%d')
+            end = datetime.strptime(c.end_date, '%Y%m%d')
+            chunk = timedelta(days=c.job_chunk)
+
+            while start <= end:
+                if (start + chunk) <= end:
+                    c.end_date = (start + chunk).strftime("%Y%m%d")
+                else:
+                    c.end_date = end.strftime("%Y%m%d")
+                print c.start_date +' bis ' + c.end_date
+
+                clist = c.to_list()
+
+                mk_jobscript(jtemplate, job_file, clist)
+
+                job_id = submit_job_to_ecserver(queue, job_file)
+                print('The job id is: ' + str(job_id.strip()))
+
+                start = start + chunk
+                c.start_date = start.strftime("%Y%m%d")
+        # submit a single job script
+        else:
+            clist = c.to_list()
+
+            mk_jobscript(jtemplate, job_file, clist)
+
+            job_id = submit_job_to_ecserver(queue, job_file)
+            print('The job id is: ' + str(job_id.strip()))
 
     else:
@@ -161 +190 @@
         mk_jobscript(jtemplate, job_file, clist)
 
-    # --------- submit the job_script to the ECMWF server
-    job_id = submit_job_to_ecserver(queue, job_file)
-    print('The job id is: ' + str(job_id.strip()))
-    print('You should get an email with subject flex.hostname.pid')
+        job_id = submit_job_to_ecserver(queue, job_file)
+        print('The job id is: ' + str(job_id.strip()))
+
+
+    print('You should get an email per job with subject flex.hostname.pid')
 
     return
@@ -223 +253 @@
     return
 
+
 if __name__ == "__main__":
     main()