- Timestamp:
- Oct 5, 2018, 3:19:05 PM (6 years ago)
- Branches:
- master, ctbto, dev
- Children:
- ceea034
- Parents:
- aa1af8c
- Location:
- python
- Files:
-
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
python/FlexpartTools.py
r68f5e1c r51f9853 98 98 help="Public mode - retrieves the public datasets") 99 99 100 parser.add_argument("--request", dest="request", default=0, 101 help="list all mars request in file mars_requests.dat \ 102 and skip submission to mars") 100 103 101 104 args = parser.parse_args() … … 188 191 if args.flexpart_root_scripts!=None: 189 192 c.flexpart_root_scripts=args.flexpart_root_scripts 193 194 # set request attribute to control file 195 if args.request != '0': 196 c.request=args.request 197 198 if c.request != '0': 199 marsfile = os.path.join(c.inputdir, 'mars_request.csv') 200 if os.path.isfile(marsfile): 201 os.remove(marsfile) 190 202 191 203 return args,c … … 564 576 self.type[i] = 0 565 577 566 578 if not hasattr(self,'request'): 579 self.request='0' 567 580 568 581 return … … 642 655 return 643 656 657 def print_infodata_csv(self, inputdir, request_number): 658 ''' 659 @Description: 660 Write all request parameter in alpabetical order into a "csv" file. 661 662 @Input: 663 self: instance of MarsRetrieval 664 For description see class documentation. 665 666 inputdir: string 667 The path where all data from the retrievals are stored. 668 669 request_number: integer 670 Number of mars requests for flux and non-flux data. 671 672 @Return: 673 <nothing> 674 ''' 675 676 # Get all class attributes and their values as a dictionary 677 attrs = vars(self) 678 del attrs['server'] 679 680 # open a file to store all requests to 681 with open(os.path.join(inputdir, 'mars_request.csv'), 'a') as f: 682 f.write(str(request_number) + ', ') 683 f.write(', '.join(str(attrs[key]) 684 for key in sorted(attrs.iterkeys()))) 685 f.write('\n') 686 687 return 688 644 689 def dataRetrieve(self): 645 690 attrs=vars(self).copy() … … 709 754 710 755 def __init__(self,c,fluxes=False): 711 # different mars types for retrieving reanalysis data for flexpart 712 756 # different mars types for retrieving reanalysis data for flexpart 757 758 # set a counter for the number of mars requests generated 759 self.mreq_count = 0 713 760 self.types=dict() 714 761 try: … … 912 959 913 960 914 def retrieve(self, server, public, dates, times, inputdir=''):961 def retrieve(self, server, public, dates, request, times, inputdir=''): 915 962 self.dates=dates 916 963 self.server=server … … 971 1018 972 1019 if self.basetime==None: 1020 # increase number of mars requests 1021 self.mreq_count += 1 973 1022 MR= MARSretrieval(self.server, self.public, dataset=self.dataset, marsclass=self.marsclass, stream=mfstream, 974 1023 type=mftype, levtype=pv[1], levelist=pv[2],resol=self.resol, gaussian=gaussian, … … 976 1025 date=mfdate, time=mftime,number=self.number,step=mfstep, expver=self.expver, param=pv[0]) 977 1026 978 MR.displayInfo() 979 MR.dataRetrieve() 1027 if request == "0": 1028 MR.display_info() 1029 MR.data_retrieve() 1030 elif request == "1": 1031 MR.print_infodata_csv(self.inputdir, self.mreq_count) 1032 elif request == "2": 1033 MR.print_infodata_csv(self.inputdir, self.mreq_count) 1034 MR.display_info() 1035 MR.data_retrieve() 980 1036 # The whole else section is only necessary for operational scripts. It could be removed 981 1037 else: # check if mars job requests fields beyond basetime. If yes eliminate those fields since they may not … … 1001 1057 maxtime=elimit-datetime.timedelta(hours=24) 1002 1058 mfdate='/'.join(('/'.join(mfdate.split('/')[:-1]),datetime.datetime.strftime(maxtime,'%Y%m%d'))) 1003 1059 # increase number of mars requests 1060 self.mreq_count += 1 1004 1061 MR= MARSretrieval(self.server, self.public, dataset=self.dataset, marsclass=self.marsclass, stream=self.stream, 1005 1062 type=mftype, levtype=pv[1], levelist=pv[2],resol=self.resol, gaussian=gaussian, … … 1007 1064 date=mfdate, time=mftime,number=self.number,step=mfstep, expver=self.expver, param=pv[0]) 1008 1065 1009 MR.displayInfo() 1010 MR.dataRetrieve() 1066 if request == "0": 1067 MR.display_info() 1068 MR.data_retrieve() 1069 elif request == "1": 1070 MR.print_infodata_csv(self.inputdir, self.mreq_count) 1071 elif request == "2": 1072 MR.print_infodata_csv(self.inputdir, self.mreq_count) 1073 MR.display_info() 1074 MR.data_retrieve() 1011 1075 1012 1076 maxtime=elimit-datetime.timedelta(hours=12) … … 1014 1078 mftime='00' 1015 1079 mftarget=self.inputdir+"/"+ftype+pk+'.'+mfdate+'.'+str(os.getppid())+'.'+str(os.getpid())+".grb" 1016 1080 # increase number of mars requests 1081 self.mreq_count += 1 1017 1082 MR= MARSretrieval(self.server, self.public, dataset=self.dataset, marsclass=self.marsclass, stream=self.stream, 1018 1083 type=mftype, levtype=pv[1], levelist=pv[2],resol=self.resol, gaussian=gaussian, 1019 1084 accuracy=self.accuracy,grid=pv[3],target=mftarget,area=area, 1020 1085 date=mfdate, time=mftime,number=self.number,step=mfstep, expver=self.expver, param=pv[0]) 1021 1022 MR.displayInfo() 1023 MR.dataRetrieve() 1086 if request == "0": 1087 MR.display_info() 1088 MR.data_retrieve() 1089 elif request == "1": 1090 MR.print_infodata_csv(self.inputdir, self.mreq_count) 1091 elif request == "2": 1092 MR.print_infodata_csv(self.inputdir, self.mreq_count) 1093 MR.display_info() 1094 MR.data_retrieve() 1024 1095 else: 1096 # increase number of mars requests 1097 self.mreq_count += 1 1025 1098 MR= MARSretrieval(self.server, self.public, dataset=self.dataset, marsclass=self.marsclass, stream=self.stream, 1026 1099 type=mftype, levtype=pv[1], levelist=pv[2],resol=self.resol, gaussian=gaussian, … … 1028 1101 date=mfdate, time=mftime,number=self.number,step=mfstep, expver=self.expver, param=pv[0]) 1029 1102 1030 MR.displayInfo() 1031 MR.dataRetrieve() 1103 if request == "0": 1104 MR.display_info() 1105 MR.data_retrieve() 1106 elif request == "1": 1107 MR.print_infodata_csv(self.inputdir, self.mreq_count) 1108 elif request == "2": 1109 MR.print_infodata_csv(self.inputdir, self.mreq_count) 1110 MR.display_info() 1111 MR.data_retrieve() 1032 1112 else: 1033 1113 maxtime=elimit-datetime.timedelta(hours=24) … … 1044 1124 else: 1045 1125 mftime=times[0] 1046 1126 # increase number of mars requests 1127 self.mreq_count += 1 1047 1128 MR= MARSretrieval(self.server, self.public, dataset=self.dataset, marsclass=self.marsclass, stream=self.stream, 1048 1129 type=mftype, levtype=pv[1], levelist=pv[2],resol=self.resol, gaussian=gaussian, … … 1050 1131 date=mfdate, time=mftime,number=self.number,step=mfstep, expver=self.expver, param=pv[0]) 1051 1132 1052 MR.displayInfo() 1053 MR.dataRetrieve() 1133 if request == "0": 1134 MR.display_info() 1135 MR.data_retrieve() 1136 elif request == "1": 1137 MR.print_infodata_csv(self.inputdir, self.mreq_count) 1138 elif request == "2": 1139 MR.print_infodata_csv(self.inputdir, self.mreq_count) 1140 MR.display_info() 1141 MR.data_retrieve() 1054 1142 1055 1143 if int(mftimesave.split('/')[0])==0 and int(mfstep.split('/')[0])==0 and pk!='OG_OROLSM__SL': … … 1058 1146 mfstep='000' 1059 1147 mftarget=self.inputdir+"/"+ftype+pk+'.'+mfdate+'.'+str(os.getppid())+'.'+str(os.getpid())+".grb" 1060 1148 # increase number of mars requests 1149 self.mreq_count += 1 1061 1150 MR= MARSretrieval(self.server, self.public, dataset=self.dataset, marsclass=self.marsclass, stream=self.stream, 1062 1151 type=mftype, levtype=pv[1], levelist=pv[2],resol=self.resol, gaussian=gaussian, … … 1064 1153 date=mfdate, time=mftime,number=self.number,step=mfstep, expver=self.expver, param=pv[0]) 1065 1154 1066 MR.displayInfo() 1067 MR.dataRetrieve() 1068 1069 1070 1071 print "MARS retrieve done... " 1155 if request == "0": 1156 MR.display_info() 1157 MR.data_retrieve() 1158 elif request == "1": 1159 MR.print_infodata_csv(self.inputdir, self.mreq_count) 1160 elif request == "2": 1161 MR.print_infodata_csv(self.inputdir, self.mreq_count) 1162 MR.display_info() 1163 MR.data_retrieve() 1164 1165 1166 if request == "0" or request == "2": 1167 print('MARS retrieve done ... ') 1168 elif request == "1": 1169 print('MARS request printed ...') 1072 1170 1073 1171 def getFlexpartTime(self, type,step, time): -
python/getMARSdata.py
re18f4b5 r51f9853 60 60 print "start date %s "%(c.start_date) 61 61 print "end date %s "%(c.end_date) 62 63 if c.request == '0' or c.request == '2': 64 print("Retrieving EC data!") 65 elif c.request == '1': 66 print("Printing mars requests!") 62 67 63 68 if ecapi: … … 91 96 92 97 datechunk=datetime.timedelta(days=int(c.date_chunk)) 93 print 'removing content of '+c.inputdir 94 tobecleaned=glob.glob(c.inputdir+'/*_acc_*.'+str(os.getppid())+'.*.grb') 95 for f in tobecleaned: 96 os.remove(f) 98 if c.request == '0' or c.request == '2': 99 print 'removing content of '+c.inputdir 100 tobecleaned=glob.glob(c.inputdir+'/*_acc_*.'+str(os.getppid())+'.*.grb') 101 for f in tobecleaned: 102 os.remove(f) 97 103 98 104 times=None … … 109 115 print "retrieve " + dates + " in dir " + c.inputdir 110 116 try: 111 flexpart.retrieve(server, c.public, dates, times, c.inputdir)117 flexpart.retrieve(server, c.public, dates, c.request, times, c.inputdir) 112 118 except IOError: 113 119 myerror(c,'MARS request failed') … … 125 131 126 132 print "retrieve " + dates + " in dir " + c.inputdir 127 flexpart.retrieve(server, c.public, dates, times, c.inputdir)133 flexpart.retrieve(server, c.public, dates, c.request, times, c.inputdir) 128 134 day+=datechunk 129 135 130 136 131 132 tobecleaned=glob.glob(c.inputdir+'/*__*.'+str(os.getppid())+'.*.grb') 133 for f in tobecleaned: 134 os.remove(f) 137 if c.request == '0' or c.request == '2': 138 print 'removing content of '+c.inputdir 139 tobecleaned=glob.glob(c.inputdir+'/*__*.'+str(os.getppid())+'.*.grb') 140 for f in tobecleaned: 141 os.remove(f) 142 135 143 day=start 136 144 times=None … … 145 153 print "retrieve " + dates + " in dir " + c.inputdir 146 154 147 flexpart.retrieve(server, c.public, dates, times, c.inputdir)155 flexpart.retrieve(server, c.public, dates, c.request, times, c.inputdir) 148 156 day+=datechunk 149 157 -
python/submit.py
re18f4b5 r51f9853 56 56 c.outputdir=os.path.join(calledfromdir,c.outputdir) 57 57 getMARSdata(args,c) 58 prepareFLEXPART(args,c) 59 normalexit(c) 58 if c.request == '0' or c.request == '2': 59 prepareFLEXPART(args,c) 60 normalexit(c) 61 else: 62 normalexit(c) 60 63 else: 61 64 submit(args.job_template,c,args.queue)
Note: See TracChangeset
for help on using the changeset viewer.