- Timestamp:
- Sep 29, 2018, 1:18:15 PM (6 years ago)
- Branches:
- master, ctbto, dev
- Children:
- 65748f4
- Parents:
- 7b4e39e
- Location:
- source/python
- Files:
-
- 6 edited
Legend:
- Unmodified
- Added
- Removed
-
source/python/_config.py
r25b14be r295ff45 32 32 # ------------------------------------------------------------------------------ 33 33 34 FILE_MARS_REQUESTS = 'mars_requests. dat'34 FILE_MARS_REQUESTS = 'mars_requests.csv' 35 35 FORTRAN_EXECUTABLE = 'CONVERT2' 36 36 FILE_USER_ENVVARS = 'ECMWF_ENV' -
source/python/classes/EcFlexpart.py
r4971f63 r295ff45 130 130 <nothing> 131 131 ''' 132 # set a counter for the number of mars requests generated 133 self.mreq_count = 0 132 134 133 135 # different mars types for retrieving data for flexpart … … 225 227 'OG__ML': '', 'OG__SL': '', 226 228 'OG_OROLSM_SL': '', 'OG_acc_SL': ''} 229 # the self.params dictionary stores a list of 230 # [param, levtype, levelist, grid] per key 227 231 228 232 if fluxes is False: … … 278 282 # stl2/stl3/stl4/swvl1/swvl2/swvl3/swvl4'.upper() 279 283 wrf_sfc = '134/235/167/165/166/168/129/172/34/31/141/ \ 280 139/170/183/236/39/40/41/42' .upper()284 139/170/183/236/39/40/41/42' 281 285 lwrt_sfc = wrf_sfc.split('/') 282 286 for par in lwrt_sfc: … … 292 296 return 293 297 298 299 def _start_retrievement(self, request, par_dict): 300 ''' 301 @Description: 302 Creates the Mars Retrieval and prints or submits the request 303 depending on the status of the request variable. 304 305 @Input: 306 self: instance of EcFlexpart 307 The current object of the class. 308 309 request: integer 310 Selects the mode of retrieval. 311 0: Retrieves the data from ECMWF. 312 1: Prints the mars requests to an output file. 313 2: Retrieves the data and prints the mars request. 314 315 par_dict: dictionary 316 Contains all parameter which have to be set for creating the 317 Mars Retrievals. The parameter are: 318 marsclass, stream, type, levtype, levelist, resol, gaussian, 319 accuracy, grid, target, area, date, time, number, step, expver, 320 param 321 322 @Return: 323 <nothing> 324 ''' 325 # increase number of mars requests 326 self.mreq_count += 1 327 328 MR = MarsRetrieval(self.server, 329 marsclass=par_dict['marsclass'], 330 stream=par_dict['stream'], 331 type=par_dict['type'], 332 levtype=par_dict['levtype'], 333 levelist=par_dict['levelist'], 334 resol=par_dict['resol'], 335 gaussian=par_dict['gaussian'], 336 accuracy=par_dict['accuracy'], 337 grid=par_dict['grid'], 338 target=par_dict['target'], 339 area=par_dict['area'], 340 date=par_dict['date'], 341 time=par_dict['time'], 342 number=par_dict['number'], 343 step=par_dict['step'], 344 expver=par_dict['expver'], 345 param=par_dict['param']) 346 347 if request == 0: 348 MR.display_info() 349 MR.data_retrieve() 350 elif request == 1: 351 MR.print_infodata_csv(self.inputdir, self.mreq_count) 352 elif request == 2: 353 MR.print_infodata_csv(self.inputdir, self.mreq_count) 354 MR.display_info() 355 MR.data_retrieve() 356 else: 357 print('Failure') 358 359 return 360 361 362 def _mk_targetname(self, ftype, param, date): 363 ''' 364 @Description: 365 Creates the filename for the requested grib data to be stored in. 366 This name is passed as the "target" parameter in the request. 367 368 @Input: 369 ftype: string 370 Shortcut name of the type of the field. E.g. AN, FC, PF, ... 371 372 param: string 373 Shortcut of the grid type. E.g. SH__ML, SH__SL, GG__ML, 374 GG__SL, OG__ML, OG__SL, OG_OROLSM_SL, OG_acc_SL 375 376 date: string 377 The date period of the grib data to be stored in this file. 378 379 @Return: 380 targetname: string 381 The target filename for the grib data. 382 ''' 383 targetname = (self.inputdir + '/' + ftype + param + '.' + date + '.' + 384 str(os.getppid()) + '.' + str(os.getpid()) + '.grb') 385 386 return targetname 294 387 295 388 def write_namelist(self, c, filename): … … 381 474 "YYYYMMDD/to/YYYYMMDD" 382 475 476 request: integer 477 Selects the mode of retrieval. 478 0: Retrieves the data from ECMWF. 479 1: Prints the mars requests to an output file. 480 2: Retrieves the data and prints the mars request. 481 383 482 inputdir: string, optional 384 483 Path to the directory where the retrieved data is about … … 393 492 oro = False 394 493 494 # define times 495 t12h = timedelta(hours=12) 496 t24h = timedelta(hours=24) 497 498 # dictionary which contains all parameter for the mars request 499 # Entries with a "None" will change for different requests and therefore 500 # will be set in each request seperately 501 retr_param_dict = {'marsclass':self.marsclass, 502 'stream':None, 503 'type':None, 504 'levtype':None, 505 'levelist':None, 506 'resol':self.resol, 507 'gaussian':None, 508 'accuracy':self.accuracy, 509 'grid':None, 510 'target':None, 511 'area':None, 512 'date':None, 513 'time':None, 514 'number':self.number, 515 'step':None, 516 'expver':self.expver, 517 'param':None} 518 395 519 for ftype in self.types: 520 # fk contains fields types such as 521 # [AN, FC, PF, CV] 522 # fv contains all of the items of the belonging key 523 # [times, steps] 396 524 for pk, pv in self.params.iteritems(): 525 # pk contains one of these keys of params 526 # [SH__ML, SH__SL, GG__ML, GG__SL, OG__ML, OG__SL, 527 # OG_OROLSM_SL, OG_acc_SL] 528 # pv contains all of the items of the belonging key 529 # [param, levtype, levelist, grid] 397 530 if isinstance(pv, str): 398 531 continue 399 mftype = '' + ftype 400 mftime = self.types[ftype]['times'] 401 mfstep = self.types[ftype]['steps'] 402 mfdate = self.dates 403 mfstream = self.stream 404 mftarget = self.inputdir + "/" + ftype + pk + '.' + \ 405 self.dates.split('/')[0] + '.' + str(os.getppid()) +\ 406 '.' + str(os.getpid()) + ".grb" 532 retr_param_dict['type'] = '' + ftype 533 retr_param_dict['time'] = self.types[ftype]['times'] 534 retr_param_dict['step'] = self.types[ftype]['steps'] 535 retr_param_dict['date'] = self.dates 536 retr_param_dict['stream'] = self.stream 537 retr_param_dict['target'] = self.inputdir + "/" + ftype + \ 538 pk + '.' + self.dates.split('/')[0] + '.' + \ 539 str(os.getppid()) + '.' + str(os.getpid()) + ".grb" 540 retr_param_dict['param'] = pv[0] 541 retr_param_dict['levtype'] = pv[1] 542 retr_param_dict['levelist'] = pv[2] 543 retr_param_dict['grid'] = pv[3] 544 retr_param_dict['area'] = self.area 545 retr_param_dict['gaussian'] = self.gaussian 546 407 547 if pk == 'OG__SL': 408 548 pass 409 if pk == 'OG_OROLSM__SL': 410 if not oro: 411 mfstream = 'OPER' 412 mftype = 'AN' 413 mftime = '00' 414 mfstep = '000' 415 mfdate = self.dates.split('/')[0] 416 mftarget = self.inputdir + "/" + pk + '.' + mfdate + \ 417 '.' + str(os.getppid()) + '.' + \ 418 str(os.getpid()) + ".grb" 419 oro = True 420 else: 421 continue 549 if pk == 'OG_OROLSM__SL' and not oro: 550 oro = True 551 retr_param_dict['stream'] = 'OPER' 552 retr_param_dict['type'] = 'AN' 553 retr_param_dict['time'] = '00' 554 retr_param_dict['step'] = '000' 555 retr_param_dict['date'] = self.dates.split('/')[0] 556 retr_param_dict['target'] = self._mk_targetname('', 557 pk, retr_param_dict['date']) 558 elif pk == 'OG_OROLSM__SL' and oro: 559 continue 422 560 if pk == 'GG__SL' and pv[0] == 'Q': 423 area = "" 424 gaussian = 'reduced' 425 else: 426 area = self.area 427 gaussian = self.gaussian 561 retr_param_dict['area'] = "" 562 retr_param_dict['gaussian'] = 'reduced' 428 563 429 564 # ------ on demand path -------------------------------------------------- 430 565 if not self.basetime: 431 MR = MarsRetrieval(self.server, 432 marsclass=self.marsclass, 433 stream=mfstream, 434 type=mftype, 435 levtype=pv[1], 436 levelist=pv[2], 437 resol=self.resol, 438 gaussian=gaussian, 439 accuracy=self.accuracy, 440 grid=pv[3], 441 target=mftarget, 442 area=area, 443 date=mfdate, 444 time=mftime, 445 number=self.number, 446 step=mfstep, 447 expver=self.expver, 448 param=pv[0]) 449 450 if request == 0: 451 MR.display_info() 452 MR.data_retrieve() 453 elif request == 1: 454 MR.print_info(self.inputdir) 455 elif request == 2: 456 MR.print_info(self.inputdir) 457 MR.display_info() 458 MR.data_retrieve() 459 else: 460 print('Failure') 566 self._start_retrievement(request, retr_param_dict) 461 567 # ------ operational path ------------------------------------------------ 462 568 else: … … 464 570 # If yes eliminate those fields since they may not 465 571 # be accessible with user's credentials 466 if 'by' in mfstep:572 if 'by' in retr_param_dict['step']: 467 573 sm1 = 2 468 574 else: 469 575 sm1 = -1 470 576 471 if 'by' in mftime:577 if 'by' in retr_param_dict['time']: 472 578 tm1 = 2 473 579 else: 474 580 tm1 = -1 475 581 476 maxdate = datetime.strptime( mfdate.split('/')[-1] +477 mftime.split('/')[tm1],582 maxdate = datetime.strptime(retr_param_dict['date'].split('/')[-1] + 583 retr_param_dict['time'].split('/')[tm1], 478 584 '%Y%m%d%H') 479 istep = int(mfstep.split('/')[sm1])480 maxtime = maxdate + timedelta(hours=istep)481 482 elimit = datetime.strptime( mfdate.split('/')[-1] +585 maxtime = maxdate + \ 586 timedelta(hours=int(retr_param_dict['step'].split('/')[sm1])) 587 588 elimit = datetime.strptime(retr_param_dict['date'].split('/')[-1] + 483 589 self.basetime, '%Y%m%d%H') 484 590 … … 492 598 # if maxtime-elimit<12h reduce step for last time 493 599 # A split of the MARS job into 2 is likely necessary. 494 maxtime = elimit - timedelta(hours=24) 495 mfdate = '/'.join(['/'.join(mfdate.split('/')[:-1]), 496 datetime.strftime(maxtime, 497 '%Y%m%d')]) 498 499 MR = MarsRetrieval(self.server, 500 marsclass=self.marsclass, 501 stream=self.stream, 502 type=mftype, 503 levtype=pv[1], 504 levelist=pv[2], 505 resol=self.resol, 506 gaussian=gaussian, 507 accuracy=self.accuracy, 508 grid=pv[3], 509 target=mftarget, 510 area=area, 511 date=mfdate, 512 time=mftime, 513 number=self.number, 514 step=mfstep, 515 expver=self.expver, 516 param=pv[0]) 517 518 MR.display_info() 519 MR.data_retrieve() 520 521 maxtime = elimit - timedelta(hours=12) 522 mfdate = datetime.strftime(maxtime, '%Y%m%d') 523 mftime = '00' 524 mftarget = self.inputdir + "/" + ftype + pk + \ 525 '.' + mfdate + '.' + str(os.getppid()) +\ 526 '.' + str(os.getpid()) + ".grb" 527 528 MR = MarsRetrieval(self.server, 529 marsclass=self.marsclass, 530 stream=self.stream, 531 type=mftype, 532 levtype=pv[1], 533 levelist=pv[2], 534 resol=self.resol, 535 gaussian=gaussian, 536 accuracy=self.accuracy, 537 grid=pv[3], 538 target=mftarget, 539 area=area, 540 date=mfdate, 541 time=mftime, 542 number=self.number, 543 step=mfstep, 544 expver=self.expver, 545 param=pv[0]) 546 547 MR.display_info() 548 549 MR.data_retrieve() 600 601 602 startdate = retr_param_dict['date'].split('/')[0] 603 enddate = datetime.strftime(elimit - t24h,'%Y%m%d') 604 retr_param_dict['date'] = '/'.join([startdate, 'to', enddate]) 605 606 self._start_retrievement(request, retr_param_dict) 607 608 retr_param_dict['date'] = \ 609 datetime.strftime(elimit - t12h, '%Y%m%d') 610 retr_param_dict['time'] = '00' 611 retr_param_dict['target'] = \ 612 self._mk_targetname(ftype, pk, 613 retr_param_dict['date']) 614 615 self._start_retrievement(request, retr_param_dict) 616 550 617 # -------------- non flux data ------------------------ 551 618 else: 552 MR = MarsRetrieval(self.server, 553 marsclass=self.marsclass, 554 stream=self.stream, 555 type=mftype, 556 levtype=pv[1], 557 levelist=pv[2], 558 resol=self.resol, 559 gaussian=gaussian, 560 accuracy=self.accuracy, 561 grid=pv[3], 562 target=mftarget, 563 area=area, 564 date=mfdate, 565 time=mftime, 566 number=self.number, 567 step=mfstep, 568 expver=self.expver, 569 param=pv[0]) 570 571 MR.display_info() 572 MR.data_retrieve() 573 else: # basetime == 0 ??? #AP 574 575 maxtime = elimit - timedelta(hours=24) 576 mfdate = datetime.strftime(maxtime, '%Y%m%d') 577 mftimesave = ''.join(mftime) 578 579 if '/' in mftime: 580 times = mftime.split('/') 581 while ((int(times[0]) + 582 int(mfstep.split('/')[0]) <= 12) and 583 (pk != 'OG_OROLSM__SL') and 'acc' not in pk): 619 self._start_retrievement(request, retr_param_dict) 620 621 else: # basetime = 0 622 retr_param_dict['date'] = \ 623 datetime.strftime(elimit - t24h, '%Y%m%d') 624 625 timesave = ''.join(retr_param_dict['time']) 626 627 if '/' in retr_param_dict['time']: 628 times = retr_param_dict['time'].split('/') 629 steps = retr_param_dict['step'].split('/') 630 while (pk != 'OG_OROLSM__SL' and 631 'acc' not in pk and 632 (int(times[0]) + int(steps[0])) <= 12): 584 633 times = times[1:] 634 585 635 if len(times) > 1: 586 mftime= '/'.join(times)636 retr_param_dict['time'] = '/'.join(times) 587 637 else: 588 mftime = times[0] 589 590 MR = MarsRetrieval(self.server, 591 marsclass=self.marsclass, 592 stream=self.stream, 593 type=mftype, 594 levtype=pv[1], 595 levelist=pv[2], 596 resol=self.resol, 597 gaussian=gaussian, 598 accuracy=self.accuracy, 599 grid=pv[3], 600 target=mftarget, 601 area=area, 602 date=mfdate, 603 time=mftime, 604 number=self.number, 605 step=mfstep, 606 expver=self.expver, 607 param=pv[0]) 608 609 MR.display_info() 610 MR.data_retrieve() 611 612 if (int(mftimesave.split('/')[0]) == 0 and 613 int(mfstep.split('/')[0]) == 0 and 614 pk != 'OG_OROLSM__SL'): 615 616 mfdate = datetime.strftime(elimit, '%Y%m%d') 617 mftime = '00' 618 mfstep = '000' 619 mftarget = self.inputdir + "/" + ftype + pk + \ 620 '.' + mfdate + '.' + str(os.getppid()) +\ 621 '.' + str(os.getpid()) + ".grb" 622 623 MR = MarsRetrieval(self.server, 624 marsclass=self.marsclass, 625 stream=self.stream, 626 type=mftype, 627 levtype=pv[1], 628 levelist=pv[2], 629 resol=self.resol, 630 gaussian=gaussian, 631 accuracy=self.accuracy, 632 grid=pv[3], 633 target=mftarget, 634 area=area, 635 date=mfdate, 636 time=mftime, 637 number=self.number, 638 step=mfstep, 639 expver=self.expver, 640 param=pv[0]) 641 642 MR.display_info() 643 MR.data_retrieve() 638 retr_param_dict['time'] = times[0] 639 640 self._start_retrievement(request, retr_param_dict) 641 642 if (pk != 'OG_OROLSM__SL' and 643 int(retr_param_dict['step'].split('/')[0]) == 0 and 644 int(timesave.split('/')[0]) == 0): 645 646 retr_param_dict['date'] = \ 647 datetime.strftime(elimit, '%Y%m%d') 648 retr_param_dict['time'] = '00' 649 retr_param_dict['step'] = '000' 650 retr_param_dict['target'] = \ 651 self._mk_targetname(ftype, pk, 652 retr_param_dict['date']) 653 654 self._start_retrievement(request, retr_param_dict) 644 655 645 656 if request == 0 or request == 2: … … 904 915 cdateH = datetime.strftime(timestamp, '%Y%m%d%H') 905 916 906 if c.basetime is not None:917 if c.basetime: 907 918 slimit = datetime.strptime(c.start_date + '00', '%Y%m%d%H') 908 919 bt = '23' … … 1242 1253 grib_write(gid, f_handle) 1243 1254 1244 if c.basetime is not None:1255 if c.basetime: 1245 1256 elimit = datetime.strptime(c.end_date + 1246 1257 c.basetime, '%Y%m%d%H') … … 1295 1306 1296 1307 return 1308 1309 1310 -
source/python/classes/MarsRetrieval.py
r25b14be r295ff45 335 335 336 336 337 def print_info(self, inputdir ):337 def print_info(self, inputdir, request_number): 338 338 ''' 339 339 @Description: … … 348 348 The path where all data from the retrievals are stored. 349 349 350 @Return: 350 request_number: integer 351 Number of mars requests for flux and non-flux data. 352 353 @Return: 351 354 <nothing> 352 355 ''' … … 357 360 with open(os.path.join(inputdir, 358 361 _config.FILE_MARS_REQUESTS), 'a') as f: 359 f.write('mars \n')362 f.write('mars_request #' + str(request_number) + '\n') 360 363 # iterate through all attributes and print them 361 364 # with their corresponding values … … 366 369 f.write(item[0] + ': ' + str(item[1]) + '\n') 367 370 f.write('\n\n') 371 372 return 373 374 375 def print_infodata_csv(self, inputdir, request_number): 376 ''' 377 @Description: 378 Write all request parameter in alpabetical order into a "csv" file. 379 380 @Input: 381 self: instance of MarsRetrieval 382 For description see class documentation. 383 384 inputdir: string 385 The path where all data from the retrievals are stored. 386 387 request_number: integer 388 Number of mars requests for flux and non-flux data. 389 390 @Return: 391 <nothing> 392 ''' 393 394 # Get all class attributes and their values as a dictionary 395 attrs = vars(self) 396 del attrs['server'] 397 398 # open a file to store all requests to 399 with open(os.path.join(inputdir, 400 _config.FILE_MARS_REQUESTS), 'a') as f: 401 f.write(str(request_number) + ', ') 402 f.write(', '.join(str(attrs[key]) 403 for key in sorted(attrs.iterkeys()))) 404 f.write('\n') 368 405 369 406 return -
source/python/mods/get_mars_data.py
r4971f63 r295ff45 67 67 ''' 68 68 @Description: 69 If get_mars_data is called from command line, this function controls 69 If get_mars_data is called directly from command line, 70 70 71 the program flow and calls the argumentparser function and 71 72 the get_mars_data function for retrieving EC data. … … 82 83 83 84 env_parameter = read_ecenv(_config.PATH_ECMWF_ENV) 84 c.assign_args_to_control(args , env_parameter)85 c.assign_args_to_control(args) 85 86 c.assign_envs_to_control(env_parameter) 86 87 c.check_conditions(args.queue) -
source/python/mods/prepare_flexpart.py
r4971f63 r295ff45 92 92 93 93 env_parameter = read_ecenv(_config.PATH_ECMWF_ENV) 94 c.assign_args_to_control(args , env_parameter)94 c.assign_args_to_control(args) 95 95 c.assign_envs_to_control(env_parameter) 96 96 c.check_conditions(args.queue) 97 97 98 prepare_flexpart(args.ppid, c) 98 99 -
source/python/submit.py
r4971f63 r295ff45 87 87 # on ECMWF server this would also be the local side 88 88 called_from_dir = os.getcwd() 89 if not args.queue:89 if args.queue is None: 90 90 if c.inputdir[0] != '/': 91 91 c.inputdir = os.path.join(called_from_dir, c.inputdir)
Note: See TracChangeset
for help on using the changeset viewer.