Changeset d4696e0 in flex_extract.git
- Timestamp:
- Jan 18, 2019, 10:50:19 PM (5 years ago)
- Branches:
- master, ctbto, dev
- Children:
- 4c1d7de
- Parents:
- 6f951ca
- Files:
-
- 1 added
- 7 edited
Legend:
- Unmodified
- Added
- Removed
-
source/python/classes/ControlFile.py
r6f951ca rd4696e0 93 93 The time step in hours. Default value is None. 94 94 95 basetime : str95 basetime : int 96 96 The time for a half day retrieval. The 12 hours upfront are to be 97 97 retrieved. Default value is None. … … 301 301 Switch to keep temporary files at the end of postprocessing (1) or 302 302 to delete all temporary files except the final output files (0). 303 Default value is 0. 304 305 oper : int 306 Switch to prepare the operational job script. Start date, end date and 307 basetime will be prepared with environment variables. 303 308 Default value is 0. 304 309 … … 409 414 self.install_target = None 410 415 self.debug = 0 416 self.oper = 0 411 417 self.request = 0 412 418 self.public = 0 … … 417 423 self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff', 418 424 'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage', 419 'ectrans', 'debug', ' request', 'public', 'purefc',420 ' rrint']425 'ectrans', 'debug', 'oper', 'request', 'public', 426 'purefc', 'rrint'] 421 427 422 428 self._read_controlfile() … … 599 605 self.end_date) 600 606 601 check_basetime(self.basetime)607 self.basetime = check_basetime(self.basetime) 602 608 603 609 self.levelist, self.level = check_levels(self.levelist, self.level) … … 624 630 self.acctype = check_acctype(self.acctype, self.type) 625 631 626 self.acctime = check_acctime(self.acctime, self. acctype, self.purefc)627 628 self.accmaxstep = check_accmaxstep(self.accmaxstep, self. acctype,632 self.acctime = check_acctime(self.acctime, self.marsclass, self.purefc) 633 634 self.accmaxstep = check_accmaxstep(self.accmaxstep, self.marsclass, 629 635 self.purefc, self.maxstep) 630 636 -
source/python/classes/EcFlexpart.py
r6f951ca rd4696e0 106 106 data. 107 107 108 basetime : str108 basetime : int 109 109 The time for a half day retrieval. The 12 hours upfront are to be 110 110 retrieved. … … 276 276 i = 0 277 277 for ty, st, ti in zip(ftype, fstep, ftime): 278 btlist = range( 24)279 if self.basetime == '12':278 btlist = range(len(ftime)) 279 if self.basetime == 12: 280 280 btlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] 281 if self.basetime == '00':281 if self.basetime == 0: 282 282 btlist = [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 0] 283 283 … … 287 287 # (int(c.time[i]) in btlist or c.purefc): 288 288 289 if (i nt(ti)in btlist) or self.purefc:289 if (i in btlist) or self.purefc: 290 290 291 291 if ((ty.upper() == 'AN' and (int(ti) % int(self.dtime)) == 0) or … … 305 305 self.types[ty]['steps'] += st 306 306 i += 1 307 307 308 return 308 309 … … 708 709 709 710 # ------ on demand path -------------------------------------------------- 710 if not self.basetime:711 if self.basetime is None: 711 712 # ******* start retrievement 712 713 self._start_retrievement(request, retr_param_dict) … … 718 719 719 720 enddate = retr_param_dict['date'].split('/')[-1] 720 elimit = datetime.strptime(enddate + s elf.basetime,721 elimit = datetime.strptime(enddate + str(self.basetime), 721 722 '%Y%m%d%H') 722 723 723 if self.basetime == '12':724 if self.basetime == 12: 724 725 # -------------- flux data ---------------------------- 725 726 if 'acc' in pk: 726 727 # Strategy:728 # if maxtime-elimit >= 24h reduce date by 1,729 # if 12h <= maxtime-elimit<12h reduce time for last date730 # if maxtime-elimit<12h reduce step for last time731 # A split of the MARS job into 2 is likely necessary.732 733 734 727 startdate = retr_param_dict['date'].split('/')[0] 735 728 enddate = datetime.strftime(elimit - t24h,'%Y%m%d') … … 756 749 self._start_retrievement(request, retr_param_dict) 757 750 758 el se: # basetime = 0751 elif self.basetime == 0: 759 752 retr_param_dict['date'] = \ 760 753 datetime.strftime(elimit - t24h, '%Y%m%d') … … 762 755 timesave = ''.join(retr_param_dict['time']) 763 756 764 if '/' in retr_param_dict['time']: 757 if ('/' in retr_param_dict['time'] and 758 pk != 'OG_OROLSM__SL' and 759 'acc' not in pk ) : 765 760 times = retr_param_dict['time'].split('/') 766 761 steps = retr_param_dict['step'].split('/') 767 while (pk != 'OG_OROLSM__SL' and 768 'acc' not in pk and 769 (int(times[0]) + int(steps[0])) <= 12): 762 print 'times', times, int(times[0]), times[1:] 763 print 'steps', steps, int(steps[0]) 764 while int(times[0]) + int(steps[0]) <= 12: 765 print 'HELLO' 770 766 times = times[1:] 771 772 if len(times) > 1: 773 retr_param_dict['time'] = '/'.join(times) 774 else: 775 retr_param_dict['time'] = times[0] 776 777 # ******* start retrievement 778 self._start_retrievement(request, retr_param_dict) 767 print 'in while 1 ', times 768 769 if len(times) > 1: 770 retr_param_dict['time'] = '/'.join(times) 771 else: 772 retr_param_dict['time'] = times[0] 773 774 print 'in while 2 ', times 775 print retr_param_dict['time'] 779 776 780 777 if (pk != 'OG_OROLSM__SL' and … … 790 787 retr_param_dict['date']) 791 788 792 # ******* start retrievement 793 self._start_retrievement(request, retr_param_dict) 789 # ******* start retrievement 790 self._start_retrievement(request, retr_param_dict) 791 else: 792 raise ValueError('ERROR: Basetime has an invalid value ' 793 '-> {}'.format(str(basetime))) 794 794 795 795 if request == 0 or request == 2: … … 985 985 t_m1dt = t_date + timedelta(hours=step-int(c.dtime)) 986 986 t_m2dt = t_date + timedelta(hours=step-2*int(c.dtime)) 987 t_enddate = None 987 if c.basetime is not None: 988 t_enddate = datetime.strptime(c.end_date + str(c.basetime), 989 '%Y%m%d%H') 990 else: 991 t_enddate = t_date + timedelta(2*int(c.dtime)) 988 992 989 993 if c.purefc: … … 1107 1111 codes_write(gid, f_handle) 1108 1112 1109 if c.basetime: 1110 t_enddate = datetime.strptime(c.end_date + c.basetime, 1111 '%Y%m%d%H') 1112 else: 1113 t_enddate = t_date + timedelta(2*int(c.dtime)) 1114 1115 # squeeze out information of last two steps 1116 # contained in deac_vals[parId] 1117 # Note that deac_vals[parId][0] has not been popped 1118 # in this case 1119 1120 if step == c.maxstep and c.purefc or \ 1121 t_dt == t_enddate: 1122 # last step 1123 if c.purefc: 1124 values = deac_vals[parId][3] 1125 codes_set_values(gid, values) 1126 codes_set(gid, 'stepRange', step) 1127 #truedatetime = t_m2dt + timedelta(hours=2*int(c.dtime)) 1128 codes_write(gid, h_handle) 1129 else: 1130 values = deac_vals[parId][3] 1131 codes_set_values(gid, values) 1132 codes_set(gid, 'stepRange', 0) 1133 truedatetime = t_m2dt + timedelta(hours=2*int(c.dtime)) 1134 codes_set(gid, 'time', truedatetime.hour * 100) 1135 codes_set(gid, 'date', int(truedatetime.strftime('%Y%m%d'))) 1136 codes_write(gid, h_handle) 1137 1138 if parId == 142 or parId == 143: 1139 values = disaggregation.darain(list(reversed(deac_vals[parId]))) 1140 else: 1141 values = disaggregation.dapoly(list(reversed(deac_vals[parId]))) 1142 1143 # step before last step 1144 if c.purefc: 1145 codes_set(gid, 'stepRange', step-int(c.dtime)) 1146 #truedatetime = t_m2dt + timedelta(hours=int(c.dtime)) 1147 codes_set_values(gid, values) 1148 codes_write(gid, g_handle) 1149 else: 1150 codes_set(gid, 'stepRange', 0) 1151 truedatetime = t_m2dt + timedelta(hours=int(c.dtime)) 1152 codes_set(gid, 'time', truedatetime.hour * 100) 1153 codes_set(gid, 'date', int(truedatetime.strftime('%Y%m%d'))) 1154 codes_set_values(gid, values) 1155 codes_write(gid, g_handle) 1113 # squeeze out information of last two steps 1114 # contained in deac_vals[parId] 1115 # Note that deac_vals[parId][0] has not been popped 1116 # in this case 1117 1118 if step == c.maxstep and c.purefc or \ 1119 t_dt == t_enddate: 1120 # last step 1121 if c.purefc: 1122 values = deac_vals[parId][3] 1123 codes_set_values(gid, values) 1124 codes_set(gid, 'stepRange', step) 1125 #truedatetime = t_m2dt + timedelta(hours=2*int(c.dtime)) 1126 codes_write(gid, h_handle) 1127 else: 1128 values = deac_vals[parId][3] 1129 codes_set_values(gid, values) 1130 codes_set(gid, 'stepRange', 0) 1131 truedatetime = t_m2dt + timedelta(hours=2*int(c.dtime)) 1132 codes_set(gid, 'time', truedatetime.hour * 100) 1133 codes_set(gid, 'date', int(truedatetime.strftime('%Y%m%d'))) 1134 codes_write(gid, h_handle) 1135 1136 if parId == 142 or parId == 143: 1137 values = disaggregation.darain(list(reversed(deac_vals[parId]))) 1138 else: 1139 values = disaggregation.dapoly(list(reversed(deac_vals[parId]))) 1140 1141 # step before last step 1142 if c.purefc: 1143 codes_set(gid, 'stepRange', step-int(c.dtime)) 1144 #truedatetime = t_m2dt + timedelta(hours=int(c.dtime)) 1145 codes_set_values(gid, values) 1146 codes_write(gid, g_handle) 1147 else: 1148 codes_set(gid, 'stepRange', 0) 1149 truedatetime = t_m2dt + timedelta(hours=int(c.dtime)) 1150 codes_set(gid, 'time', truedatetime.hour * 100) 1151 codes_set(gid, 'date', int(truedatetime.strftime('%Y%m%d'))) 1152 codes_set_values(gid, values) 1153 codes_write(gid, g_handle) 1156 1154 1157 1155 codes_release(gid) … … 1417 1415 # if the timestamp is out of basetime start/end date period, 1418 1416 # skip this specific product 1419 if c.basetime: 1420 start_time = datetime.strptime(c.end_date + c.basetime, 1417 if c.basetime is not None: 1418 time_delta = timedelta(hours=12-int(c.dtime)) 1419 start_time = datetime.strptime(c.end_date + str(c.basetime), 1421 1420 '%Y%m%d%H') - time_delta 1422 end_time = datetime.strptime(c.end_date + c.basetime,1421 end_time = datetime.strptime(c.end_date + str(c.basetime), 1423 1422 '%Y%m%d%H') 1424 1423 if timestamp < start_time or timestamp > end_time: -
source/python/mods/checks.py
r6f951ca rd4696e0 587 587 Parameters 588 588 ---------- 589 basetime : str589 basetime : int or str or None 590 590 The time for a half day retrieval. The 12 hours upfront are to be 591 591 retrieved. … … 593 593 Return 594 594 ------ 595 596 ''' 597 if basetime: 598 if int(basetime) != 0 and int(basetime) != 12: 595 basetime : int or None 596 The time for a half day retrieval. The 12 hours upfront are to be 597 retrieved. 598 ''' 599 if basetime is not None: 600 basetime = int(basetime) 601 if basetime != 0 and basetime != 12: 599 602 raise ValueError('ERROR: Basetime has an invalid value ' 600 603 '-> {}'.format(str(basetime))) 601 return 604 return basetime 602 605 603 606 def check_request(request, marsfile): … … 667 670 try: 668 671 if len(ftype) == 1 and ftype[0] != 'AN': 669 print(' Use same field type as for the non-flux fields.')672 print('... Use same field type as for the non-flux fields.') 670 673 acctype = ftype[0] 671 674 elif len(ftype) > 1 and ftype[1] != 'AN': 672 print(' Use old setting by using TYPE[1] for flux forecast!')675 print('... Use old setting by using TYPE[1] for flux forecast!') 673 676 acctype = ftype[1] 674 677 except: … … 681 684 682 685 683 def check_acctime(acctime, acctype, purefc):686 def check_acctime(acctime, marsclass, purefc): 684 687 '''Guarantees that the accumulation forecast times were set. 685 688 686 If it is not set, it is tried to set the value foresome of the689 If it is not set, it tries to set the value for some of the 687 690 most commonly used data sets. Otherwise it raises an error. 688 691 … … 692 695 The starting time from the accumulated forecasts. 693 696 694 acctype: str695 The field type for the accumulated forecast fields.697 marsclass : str 698 ECMWF data classification identifier. 696 699 697 700 purefc : int … … 703 706 The starting time from the accumulated forecasts. 704 707 ''' 708 705 709 if not acctime: 706 710 print('... Control parameter ACCTIME was not defined.') 707 711 print('... Value will be set depending on field type:\n ' 708 '\t\t EA=06/18\n\t\t EI/OD=00/12\n\t\t 709 if acctype.upper() == 'EA': # Era 5712 '\t\t EA=06/18\n\t\t EI/OD=00/12\n\t\t EP=18') 713 if marsclass.upper() == 'EA': # Era 5 710 714 acctime = '06/18' 711 elif acctype.upper() == 'EI': # Era-Interim715 elif marsclass.upper() == 'EI': # Era-Interim 712 716 acctime = '00/12' 713 elif acctype.upper() == 'EP': # CERA717 elif marsclass.upper() == 'EP': # CERA 714 718 acctime = '18' 715 elif acctype.upper() == 'OD' and not purefc: # On-demand operational719 elif marsclass.upper() == 'OD' and not purefc: # On-demand 716 720 acctime = '00/12' 717 721 else: … … 720 724 return acctime 721 725 722 def check_accmaxstep(accmaxstep, acctype, purefc, maxstep):726 def check_accmaxstep(accmaxstep, marsclass, purefc, maxstep): 723 727 '''Guarantees that the accumulation forecast step were set. 724 728 … … 728 732 The maximum forecast step for the accumulated forecast fields. 729 733 730 acctype: str731 The field type for the accumulated forecast fields.734 marsclass : str 735 ECMWF data classification identifier. 732 736 733 737 purefc : int … … 746 750 print('... Control parameter ACCMAXSTEP was not defined.') 747 751 print('... Value will be set depending on field type/time: ' 748 '\ t\t EA/EI/OD=12\n\t\tEP=24')749 if acctype.upper() in ['EA', 'EI', 'OD'] and not purefc:752 '\n\t\t EA/EI/OD=12\n\t\t EP=24') 753 if marsclass.upper() in ['EA', 'EI', 'OD'] and not purefc: 750 754 # Era 5, Era-Interim, On-demand operational 751 755 accmaxstep = '12' 752 elif acctype.upper() == 'EP': # CERA756 elif marsclass.upper() == 'EP': # CERA 753 757 accmaxstep = '18' 754 758 elif purefc and accmaxstep != maxstep: -
source/python/mods/get_mars_data.py
r6f951ca rd4696e0 252 252 chunk = timedelta(days=int(c.date_chunk)) 253 253 254 if c.basetime: 255 if c.basetime == '00': 256 start = start - timedelta(days=1) 257 258 if not c.purefc and fluxes: 254 if c.basetime == 0: 255 start = start - timedelta(days=1) 256 257 if not c.purefc and fluxes and not c.basetime == 0: 259 258 start = start - timedelta(days=1) 260 259 end = end + timedelta(days=1) -
source/python/mods/prepare_flexpart.py
r6f951ca rd4696e0 152 152 # since we need the 12 hours upfront 153 153 # (the day before from 12 UTC to current day 00 UTC) 154 if c.basetime == '00':154 if c.basetime == 0: 155 155 start = start - datetime.timedelta(days=1) 156 156 -
source/python/mods/tools.py
r6f951ca rd4696e0 147 147 parser.add_argument("--basetime", dest="basetime", 148 148 type=none_or_int, default=None, 149 help="base such as 0 0or 12 (for half day retrievals)")149 help="base such as 0 or 12 (for half day retrievals)") 150 150 parser.add_argument("--step", dest="step", 151 151 type=none_or_str, default=None, … … 162 162 type=none_or_int, default=None, 163 163 help="debug mode - leave temporary files intact") 164 parser.add_argument("--oper", dest="oper", 165 type=none_or_int, default=None, 166 help="operational mode - prepares dates with \ 167 environment variables") 164 168 parser.add_argument("--request", dest="request", 165 169 type=none_or_int, default=None, -
source/python/submit.py
r6f951ca rd4696e0 146 146 ''' 147 147 148 if not c. basetime:148 if not c.oper: 149 149 # --------- create on demand job script ------------------------------------ 150 150 if c.purefc: … … 196 196 c.start_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}' 197 197 c.end_date = '${MSJ_YEAR}${MSJ_MONTH}${MSJ_DAY}' 198 c.base _time = '${MSJ_BASETIME}'198 c.basetime = '${MSJ_BASETIME}' 199 199 if c.maxstep > 24: 200 200 c.time = '${MSJ_BASETIME} {MSJ_BASETIME}'
Note: See TracChangeset
for help on using the changeset viewer.