Changeset f2616a3 in flex_extract.git


Ignore:
Timestamp:
Dec 14, 2018, 1:03:35 PM (8 months ago)
Author:
Anne Philipp <anne.philipp@…>
Branches:
dev
Children:
0629ba8
Parents:
1eca806
Message:

implemented a job split with a new parameter 'job_chunk' so that huge time periods can automatically be splitted

Location:
source/python
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • source/python/classes/ControlFile.py

    r87ae9a3 rf2616a3  
    6868                         check_acctime, check_accmaxstep, check_time,
    6969                         check_logicals_type, check_len_type_time_step,
    70                          check_addpar)
     70                         check_addpar, check_job_chunk)
    7171
    7272# ------------------------------------------------------------------------------
     
    383383        self.addpar = check_addpar(self.addpar)
    384384
     385        self.job_chunk = check_job_chunk(self.job_chunk)
    385386
    386387        return
  • source/python/mods/checks.py

    r45fc9b4 rf2616a3  
    796796    return addpar
    797797
     798
     799def check_job_chunk(job_chunk):
     800    '''Checks that the job chunk number is positive and non zero.
     801
     802    Parameters
     803    ----------
     804    job_chunk : :obj:`integer`
     805        The number of days for a single job script.
     806
     807    Return
     808    ------
     809    job_chunk : :obj:`integer`
     810        The number of days for a single job script.
     811    '''
     812    if job_chunk < 0:
     813        raise ValueError('ERROR: The number of job chunk is negative!\n'
     814                         'It has to be a positive number!')
     815    elif job_chunk == 0:
     816        job_chunk = None
     817    else:
     818        pass
     819
     820    return job_chunk
  • source/python/mods/tools.py

    rbf48c8a rf2616a3  
    128128                        type=none_or_int, default=None,
    129129                        help="# of days to be retrieved at once")
     130    parser.add_argument("--job_chunk", dest="job_chunk",
     131                        type=none_or_int, default=None,
     132                        help="# of days to be retrieved within a single job")
    130133    parser.add_argument("--controlfile", dest="controlfile",
    131134                        type=none_or_str, default='CONTROL.temp',
  • source/python/submit.py

    r9aefaad rf2616a3  
    4848import inspect
    4949import collections
     50from datetime import datetime, timedelta
    5051
    5152# software specific classes and modules from flex_extract
     
    140141                                jtemplate[:-5] + '.ksh')
    141142
    142         clist = c.to_list()
    143 
    144         mk_jobscript(jtemplate, job_file, clist)
     143        # divide time periode into specified number of job chunks
     144        # to have multiple job scripts
     145        if c.job_chunk:
     146            start = datetime.strptime(c.start_date, '%Y%m%d')
     147            end = datetime.strptime(c.end_date, '%Y%m%d')
     148            chunk = timedelta(days=c.job_chunk)
     149
     150            while start <= end:
     151                if (start + chunk) <= end:
     152                    c.end_date = (start + chunk).strftime("%Y%m%d")
     153                else:
     154                    c.end_date = end.strftime("%Y%m%d")
     155                print c.start_date +' bis ' + c.end_date
     156
     157                clist = c.to_list()
     158
     159                mk_jobscript(jtemplate, job_file, clist)
     160
     161                job_id = submit_job_to_ecserver(queue, job_file)
     162                print('The job id is: ' + str(job_id.strip()))
     163
     164                start = start + chunk
     165                c.start_date = start.strftime("%Y%m%d")
     166        # submit a single job script
     167        else:
     168            clist = c.to_list()
     169
     170            mk_jobscript(jtemplate, job_file, clist)
     171
     172            job_id = submit_job_to_ecserver(queue, job_file)
     173            print('The job id is: ' + str(job_id.strip()))
    145174
    146175    else:
     
    161190        mk_jobscript(jtemplate, job_file, clist)
    162191
    163     # --------- submit the job_script to the ECMWF server
    164     job_id = submit_job_to_ecserver(queue, job_file)
    165     print('The job id is: ' + str(job_id.strip()))
    166     print('You should get an email with subject flex.hostname.pid')
     192        job_id = submit_job_to_ecserver(queue, job_file)
     193        print('The job id is: ' + str(job_id.strip()))
     194
     195
     196    print('You should get an email per job with subject flex.hostname.pid')
    167197
    168198    return
     
    223253    return
    224254
     255
    225256if __name__ == "__main__":
    226257    main()
Note: See TracChangeset for help on using the changeset viewer.
hosted by ZAMG