Changeset 45fc9b4 in flex_extract.git
- Timestamp:
- Dec 13, 2018, 11:44:05 PM (5 years ago)
- Branches:
- master, ctbto, dev
- Children:
- 83b8e3c
- Parents:
- c2c5948
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
source/python/mods/checks.py
ra55ac71 r45fc9b4 26 26 # ------------------------------------------------------------------------------ 27 27 28 import os 28 29 import _config 30 import exceptions 31 from tools import my_error, silent_remove 32 from datetime import datetime 29 33 # ------------------------------------------------------------------------------ 30 34 # FUNCTIONS 31 35 # ------------------------------------------------------------------------------ 32 36 37 def check_logicals_type(c, logicals): 38 '''Check that the logical variables have correct type integer. 39 40 Parameters 41 ---------- 42 c : :obj:`ControlFile` 43 Contains all the parameters of CONTROL file and 44 command line. 45 46 logicals : :obj:`list` of (:obj:`string` or :obj:`integer`) 47 Names of the switches that are used to control the flow of the 48 program. 49 50 Return 51 ------ 52 53 ''' 54 55 for var in logicals: 56 if not isinstance(getattr(c, var), int): 57 setattr(c, var, int(getattr(c, var))) 58 59 return 33 60 34 61 def check_grid(grid): … … 82 109 ---------- 83 110 grid : :obj:`string` 84 Contains grid information 111 Contains grid information. 112 113 area : :obj:`string` 114 Contains area informtion. 115 116 upper : :obj:`string` 117 The northern most latitude. 118 119 lower : :obj:`string` 120 The souther most latitude. 121 122 left : :obj:`string` 123 The western most longitude. 124 125 right : :obj:`string` 126 The eastern most longiude. 85 127 86 128 Return … … 99 141 100 142 # determine area format 101 if ( abs(float(upper) / 1000.) >= 0.05and102 abs(float(lower) / 1000.) >= 0.05and103 abs(float(left) / 1000.) >= 0.05and104 abs(float(right) / 1000.) >= 0.05):143 if ((abs(float(upper) / 10000.) >= 0.01 or float(upper) / 1000. == 0. ) and 144 (abs(float(lower) / 10000.) >= 0.01 or float(lower) / 1000. == 0. ) and 145 (abs(float(left) / 10000.) >= 0.01 or float(left) / 1000. == 0. ) and 146 (abs(float(right) / 10000.) >= 0.01 or float(right) / 1000. == 0.)): 105 147 # area is defined in 1/1000 degrees; old format 106 148 area = '{}/{}/{}/{}'.format(float(upper) / 1000., … … 108 150 float(lower) / 1000., 109 151 float(right) / 1000.) 110 elif (abs(float(upper) / 1000 .) < 0.05 and111 abs(float(lower) / 1000 .) < 0.05 and112 abs(float(left) / 1000 .) < 0.05 and113 abs(float(right) / 1000 .) < 0.05):152 elif (abs(float(upper) / 10000.) < 0.05 and 153 abs(float(lower) / 10000.) < 0.05 and 154 abs(float(left) / 10000.) < 0.05 and 155 abs(float(right) / 10000.) < 0.05): 114 156 # area is already in new format 115 157 area = '{}/{}/{}/{}'.format(float(upper), … … 200 242 201 243 202 def check_purefc( type):244 def check_purefc(ftype): 203 245 '''Check for a pure forecast mode. 204 246 205 247 Parameters 206 248 ---------- 207 type : :obj:`list` of :obj:`string`249 ftype : :obj:`list` of :obj:`string` 208 250 List of field types. 209 251 … … 215 257 ''' 216 258 217 if 'AN' not in type and '4V' not intype:259 if 'AN' not in ftype and '4V' not in ftype: 218 260 # pure forecast 219 return True 220 221 return False 222 223 224 def check_(): 225 ''' 226 227 Parameters 228 ---------- 229 par : :obj:`` 230 ... 231 232 Return 233 ------ 234 235 ''' 261 return 1 262 263 return 0 264 265 266 def check_step(step, mailfail): 267 '''Checks on step format and convert into a list of steps. 268 269 If the steps were defined with "to" and "by" they are converted into 270 a list of steps. If the steps were set in a string, it is 271 converted into a list. 272 273 Parameters 274 ---------- 275 step : :obj:`list` of :obj:`string` or :obj:`string` 276 Specifies the forecast time step from forecast base time. 277 Valid values are hours (HH) from forecast base time. 278 279 mailfail : :obj:`list` of :obj:``string` 280 Contains all email addresses which should be notified. 281 It might also contain just the ecmwf user name which will trigger 282 mailing to the associated email address for this user. 283 284 Return 285 ------ 286 step : :obj:`list` of :obj:`string` 287 List of forecast steps in format e.g. [001, 002, ...] 288 ''' 289 290 if '/' in step: 291 steps = step.split('/') 292 if 'to' in step.lower() and 'by' in step.lower(): 293 ilist = np.arange(int(steps[0]), 294 int(steps[2]) + 1, 295 int(steps[4])) 296 step = ['{:0>3}'.format(i) for i in ilist] 297 elif 'to' in step.lower() and 'by' not in step.lower(): 298 my_error(mailfail, step + ':\n' + 299 'if "to" is used in steps parameter, ' 300 'please use "by" as well') 301 else: 302 step = steps 303 304 if not isinstance(step, list): 305 step = [step] 306 307 return step 308 309 def check_type(ftype, steps): 310 '''Check if type variable is of type list and if analysis field has 311 forecast step 0. 312 313 Parameters 314 ---------- 315 ftype : :obj:`list` of :obj:`string` or :obj:`string` 316 List of field types. 317 318 steps : :obj:`string` 319 Specifies the forecast time step from forecast base time. 320 Valid values are hours (HH) from forecast base time. 321 322 Return 323 ------ 324 ftype : :obj:`list` of :obj:`string` 325 List of field types. 326 ''' 327 if not isinstance(ftype, list): 328 ftype = [ftype] 329 330 for i, val in enumerate(ftype): 331 if ftype[i] == 'AN' and int(steps[i]) != 0: 332 print('Analysis retrievals must have STEP = 0 (now set to 0)') 333 ftype[i] = 0 334 335 return ftype 336 337 def check_time(ftime): 338 '''Check if time variable is of type list. Otherwise convert to list. 339 340 Parameters 341 ---------- 342 ftime : :obj:`list` of :obj:`string` or :obj:`string` 343 The time in hours of the field. 344 345 Return 346 ------ 347 ftime : :obj:`list` of :obj:`string` 348 The time in hours of the field. 349 ''' 350 if not isinstance(ftime, list): 351 ftime = [ftime] 352 353 return ftime 354 355 def check_len_type_time_step(ftype, ftime, steps, maxstep, purefc): 356 '''Check if 357 358 Parameters 359 ---------- 360 ftype : :obj:`list` of :obj:`string` 361 List of field types. 362 363 ftime : :obj:`list` of :obj:`string` or :obj:`string` 364 The time in hours of the field. 365 366 steps : :obj:`string` 367 Specifies the forecast time step from forecast base time. 368 Valid values are hours (HH) from forecast base time. 369 370 maxstep : :obj:`integer` 371 The maximum forecast time step in hours from the forecast base time. 372 This is the maximum step for non flux (accumulated) forecast data. 373 374 purefc : :obj:`integer` 375 Switch for definition of pure forecast mode or not. 376 377 Return 378 ------ 379 ftype : :obj:`list` of :obj:`string` 380 List of field types. 381 382 ftime : :obj:`list` of :obj:`string` 383 The time in hours of the field. 384 385 steps : :obj:`string` 386 Specifies the forecast time step from forecast base time. 387 Valid values are hours (HH) from forecast base time. 388 ''' 389 if not (len(ftype) == len(ftime) == len(steps)): 390 raise ValueError('ERROR: The number of field types, times and steps ' 391 'are not the same! Please check the setting in the ' 392 'CONTROL file!') 393 394 # if pure forecast is selected and only one field type/time is set 395 # prepare a complete list of type/time/step combination upto maxstep 396 if len(ftype) == 1 and purefc: 397 ftype = [] 398 steps = [] 399 ftime = [] 400 for i in range(0, maxstep + 1): 401 ftype.append(ftype[0]) 402 steps.append('{:0>3}'.format(i)) 403 ftime.append(ftime[0]) 404 405 return ftype, ftime, steps 406 407 def check_mail(mail): 408 '''Check the string of mail addresses, seperate them and convert to a list. 409 410 Parameters 411 ---------- 412 mail : :obj:`list` of :obj:`string` or :obj:`string` 413 Contains email addresses for notifications. 414 It might also contain just the ecmwf user name which will trigger 415 mailing to the associated email address for this user. 416 417 Return 418 ------ 419 mail : :obj:`list` of :obj:``string` 420 Contains email addresses for notifications. 421 It might also contain just the ecmwf user name which will trigger 422 mailing to the associated email address for this user. 423 424 ''' 425 if not isinstance(mail, list): 426 if ',' in mail: 427 mail = mail.split(',') 428 elif ' ' in mail: 429 mail = mail.split() 430 else: 431 mail = [mail] 432 433 return mail 434 435 def check_queue(queue, gateway, destination, ecuid, ecgid): 436 '''Check if the necessary ECMWF parameters are set if the queue is 437 one of the QUEUES_LIST (in _config). 438 439 Parameters 440 ---------- 441 queue : :obj:`string` 442 Name of the queue if submitted to the ECMWF servers. 443 Used to check if ecuid, ecgid, gateway and destination 444 are set correctly and are not empty. 445 446 gateway : :obj:`string` 447 The address of the gateway server. 448 449 destination : :obj:`string` 450 The name of the destination of the gateway server for data 451 transfer through ectrans. E.g. name@genericSftp 452 453 ecuid : :obj:`string` 454 ECMWF user id. 455 456 ecgid : :obj:`string` 457 ECMWF group id. 458 459 Return 460 ------ 461 462 ''' 463 if queue in _config.QUEUES_LIST and \ 464 not gateway or not destination or \ 465 not ecuid or not ecgid: 466 raise ValueError('\nEnvironment variables GATEWAY, DESTINATION, ECUID ' 467 'and ECGID were not set properly! \n ' 468 'Please check for existence of file "ECMWF_ENV" ' 469 'in the run directory!') 236 470 return 471 472 def check_pathes(idir, odir, fpdir, fedir): 473 '''Check if output and flexpart pathes are set. 474 475 Parameters 476 ---------- 477 idir : :obj:`string` 478 Path to the temporary directory for MARS retrieval data. 479 480 odir : :obj:`string` 481 Path to the final output directory where the FLEXPART input files 482 will be stored. 483 484 fpdir : :obj:`string` 485 Path to FLEXPART root directory. 486 487 fedir : :obj:`string` 488 Path to flex_extract root directory. 489 490 Return 491 ------ 492 odir : :obj:`string` 493 Path to the final output directory where the FLEXPART input files 494 will be stored. 495 496 fpdir : :obj:`string` 497 Path to FLEXPART root directory. 498 499 ''' 500 if not fpdir: 501 fpdir = fedir 502 503 if not odir: 504 odir = idir 505 506 return odir, fpdir 507 508 def check_dates(start, end): 509 '''Checks if there is at least a start date for a one day retrieval. 510 511 Checks if end date lies after start date and end date is set. 512 513 Parameters 514 ---------- 515 start : :obj:`string` 516 The start date of the retrieval job. 517 518 end : :obj:`string` 519 The end date of the retrieval job. 520 521 Return 522 ------ 523 start : :obj:`string` 524 The start date of the retrieval job. 525 526 end : :obj:`string` 527 The end date of the retrieval job. 528 529 ''' 530 # check for having at least a starting date 531 # otherwise program is not allowed to run 532 if not start: 533 raise ValueError('start_date was neither specified in command line nor ' 534 'in CONTROL file.\n' 535 'Try "{} -h" to print usage information' 536 .format(sys.argv[0].split('/')[-1]) ) 537 538 # retrieve just one day if end_date isn't set 539 if not end: 540 end = start 541 542 dstart = datetime.strptime(start, '%Y%m%d') 543 dend = datetime.strptime(end, '%Y%m%d') 544 if dstart > dend: 545 raise ValueError('ERROR: Start date is after end date! \n' 546 'Please adapt the dates in CONTROL file or ' 547 'command line! (start={}; end={})'.format(start, end)) 548 549 return start, end 550 551 def check_maxstep(maxstep, steps): 552 '''Convert maxstep into integer if it is already given. Otherwise, select 553 maxstep by going through the steps list. 554 555 Parameters 556 ---------- 557 maxstep : :obj:`string` 558 The maximum forecast time step in hours from the forecast base time. 559 This is the maximum step for non flux (accumulated) forecast data. 560 561 steps : :obj:`string` 562 Specifies the forecast time step from forecast base time. 563 Valid values are hours (HH) from forecast base time. 564 565 Return 566 ------ 567 maxstep : :obj:`integer` 568 The maximum forecast time step in hours from the forecast base time. 569 This is the maximum step for non flux (accumulated) forecast data. 570 571 ''' 572 # if maxstep wasn't provided 573 # search for it in the "step" parameter 574 if not maxstep: 575 maxstep = 0 576 for s in steps: 577 if int(s) > maxstep: 578 maxstep = int(s) 579 else: 580 maxstep = int(maxstep) 581 582 return maxstep 583 584 def check_basetime(basetime): 585 '''Check if basetime is set and contains one of the two 586 possible values (0, 12). 587 588 Parameters 589 ---------- 590 basetime : :obj:`` 591 The time for a half day retrieval. The 12 hours upfront are to be 592 retrieved. 593 594 Return 595 ------ 596 597 ''' 598 if basetime: 599 if int(basetime) != 0 and int(basetime) != 12: 600 raise ValueError('ERROR: Basetime has an invalid value ' 601 '-> {}'.format(str(basetime))) 602 return 603 604 def check_request(request, marsfile): 605 '''Check if there is an old mars request file and remove it. 606 607 Parameters 608 ---------- 609 request : :obj:`integer` 610 Selects the mode of retrieval. 611 0: Retrieves the data from ECMWF. 612 1: Prints the mars requests to an output file. 613 2: Retrieves the data and prints the mars request. 614 615 marsfile : :obj:`string` 616 Path to the mars request file. 617 618 Return 619 ------ 620 621 ''' 622 if request != 0: 623 if os.path.isfile(marsfile): 624 silent_remove(marsfile) 625 return 626 627 def check_public(public, dataset): 628 '''Check wether the dataset parameter is set for a 629 public data set retrieval. 630 631 Parameters 632 ---------- 633 public : :obj:`ìnteger` 634 Specifies if public data are to be retrieved or not. 635 636 dataset : :obj:`string` 637 Specific name which identifies the public dataset. 638 639 Return 640 ------ 641 642 ''' 643 if public and not dataset: 644 raise ValueError('ERROR: If public mars data wants to be retrieved, ' 645 'the "dataset"-parameter has to be set too!') 646 return 647 648 def check_acctype(acctype, ftype): 649 '''Guarantees that the accumulation field type is set. 650 651 If not set, it is derivated as in the old method (TYPE[1]). 652 653 Parameters 654 ---------- 655 acctype : :obj:`string` 656 The field type for the accumulated forecast fields. 657 658 ftype : :obj:`list` of :obj:`string` 659 List of field types. 660 661 Return 662 ------ 663 acctype : :obj:`string` 664 The field type for the accumulated forecast fields. 665 ''' 666 if not acctype: 667 print('... Control parameter ACCTYPE was not defined.') 668 try: 669 if len(ftype) == 1 and ftype[0] != 'AN': 670 print('Use same field type as for the non-flux fields.') 671 acctype = ftype[0] 672 elif len(ftype) > 1 and ftype[1] != 'AN': 673 print('Use old setting by using TYPE[1] for flux forecast!') 674 acctype = ftype[1] 675 except: 676 raise ValueError('ERROR: Accumulation field type could not be set!') 677 else: 678 if acctype.upper() == 'AN': 679 raise ValueError('ERROR: Accumulation forecast fields can not be ' 680 'of type "analysis"!') 681 return acctype 682 683 684 def check_acctime(acctime, acctype, purefc): 685 '''Guarantees that the accumulation forecast times were set. 686 687 If it is not set, it is tried to set the value fore some of the 688 most commonly used data sets. Otherwise it raises an error. 689 690 Parameters 691 ---------- 692 acctime : :obj:`string` 693 The starting time from the accumulated forecasts. 694 695 acctype : :obj:`string` 696 The field type for the accumulated forecast fields. 697 698 purefc : :obj:`integer` 699 Switch for definition of pure forecast mode or not. 700 701 Return 702 ------ 703 acctime : :obj:`string` 704 The starting time from the accumulated forecasts. 705 ''' 706 if not acctime: 707 print('... Control parameter ACCTIME was not defined.') 708 print('... Value will be set depending on field type: ' 709 '\t\t EA=06/18\n\t\t EI/OD=00/12\n\t\t EP=18') 710 if acctype.upper() == 'EA': # Era 5 711 acctime = '06/18' 712 elif acctype.upper() == 'EI': # Era-Interim 713 acctime = '00/12' 714 elif acctype.upper() == 'EP': # CERA 715 acctime = '18' 716 elif acctype.upper() == 'OD' and not purefc: # On-demand operational 717 acctime = '00/12' 718 else: 719 raise ValueError('ERROR: Accumulation forecast time can not ' 720 'automatically be derived!') 721 return acctime 722 723 def check_accmaxstep(accmaxstep, acctype, purefc, maxstep): 724 '''Guarantees that the accumulation forecast step were set. 725 726 Parameters 727 ---------- 728 accmaxstep : :obj:`string` 729 The maximum forecast step for the accumulated forecast fields. 730 731 acctype : :obj:`string` 732 The field type for the accumulated forecast fields. 733 734 purefc : :obj:`integer` 735 Switch for definition of pure forecast mode or not. 736 737 maxstep : :obj:`string` 738 The maximum forecast time step in hours from the forecast base time. 739 This is the maximum step for non flux (accumulated) forecast data. 740 741 Return 742 ------ 743 accmaxstep : :obj:`string` 744 The maximum forecast step for the accumulated forecast fields. 745 ''' 746 if not accmaxstep: 747 print('... Control parameter ACCMAXSTEP was not defined.') 748 print('... Value will be set depending on field type/time: ' 749 '\t\t EA/EI/OD=12\n\t\t EP=24') 750 if acctype.upper() in ['EA', 'EI', 'OD'] and not purefc: 751 # Era 5, Era-Interim, On-demand operational 752 accmaxstep = '12' 753 elif acctype.upper() == 'EP': # CERA 754 accmaxstep = '18' 755 elif purefc and accmaxstep != maxstep: 756 accmaxstep = maxstep 757 print('... For pure forecast mode, the accumulated forecast must ' 758 'have the same maxstep as the normal forecast fields!\n' 759 '\t\t Accmaxstep was set to maxstep!') 760 else: 761 raise ValueError('ERROR: Accumulation forecast step can not ' 762 'automatically be derived!') 763 else: 764 if purefc and int(accmaxstep) != int(maxstep): 765 accmaxstep = maxstep 766 print('... For pure forecast mode, the accumulated forecast must ' 767 'have the same maxstep as the normal forecast fields!\n' 768 '\t\t Accmaxstep was set to maxstep!') 769 return accmaxstep 770 771 def check_addpar(addpar): 772 '''Check that addpar has correct format of additional parameters in 773 a single string, so that it can be easily appended to the hard coded 774 parameters that are retrieved in any case. 775 776 Parameters 777 ---------- 778 addpar : :obj:`string` or :obj:'list' of :obj:'string' 779 List of additional parameters to be retrieved. 780 781 Return 782 ------ 783 addpar : :obj:'string' 784 List of additional parameters to be retrieved. 785 ''' 786 787 if addpar and isinstance(addpar, str): 788 if '/' in addpar: 789 parlist = addpar.split('/') 790 parlist = [p for p in parlist if p is not ''] 791 else: 792 parlist = [addpar] 793 794 addpar = '/' + '/'.join(parlist) 795 796 return addpar 797
Note: See TracChangeset
for help on using the changeset viewer.