source: flex_extract.git/source/python/mods/checks.py @ e811e1a

ctbtodev
Last change on this file since e811e1a was 5551626, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

bug fix in check of length of type, time, step

  • Property mode set to 100644
File size: 25.1 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#*******************************************************************************
4# @Author: Anne Philipp (University of Vienna)
5#
6# @Date: November 2018
7#
8# @Change History:
9#
10# @License:
11#    (C) Copyright 2014-2019.
12#    Anne Philipp, Leopold Haimberger
13#
14#    This work is licensed under the Creative Commons Attribution 4.0
15#    International License. To view a copy of this license, visit
16#    http://creativecommons.org/licenses/by/4.0/ or send a letter to
17#    Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
18#*******************************************************************************
19'''This module contains check methods for the CONTROL paramaters.
20'''
21
22# ------------------------------------------------------------------------------
23# MODULES
24# ------------------------------------------------------------------------------
25
26import os
27import sys
28import _config
29import exceptions
30from tools import my_error, silent_remove
31from datetime import datetime
32import numpy as np
33# ------------------------------------------------------------------------------
34# FUNCTIONS
35# ------------------------------------------------------------------------------
36
37def check_logicals_type(c, logicals):
38    '''Check that the logical variables have correct type integer.
39
40    Parameters
41    ----------
42    c : ControlFile
43        Contains all the parameters of CONTROL file and
44        command line.
45
46    logicals : list of (str or int)
47        Names of the switches that are used to control the flow of the
48        program.
49
50    Return
51    ------
52
53    '''
54
55    for var in logicals:
56        if not isinstance(getattr(c, var), int):
57            setattr(c, var, int(getattr(c, var)))
58
59    return
60
61def check_grid(grid):
62    '''Convert grid into correct Lat/Lon format. E.g. '0.5/0.5'
63
64    Checks on format of original grid. Wether it is in the order of 1000 or 1.
65    Convert to correct grid format and substitute into "Lat/Lon" format string.
66
67    Parameters
68    ----------
69    grid : str
70        Contains grid information
71
72    Return
73    ------
74    grid : str
75        Contains grid in format Lat/lon. E.g. 0.1/0.1
76    '''
77
78    if 'N' in grid:
79        return grid
80    if '/' in grid:
81        gridx, gridy = grid.split('/')
82        if gridx == gridy:
83            grid = gridx
84        else:
85            raise ValueError('GRID parameter contains two values '
86                             'which are unequal %s' (grid))
87    # determine grid format
88    if float(grid) / 100. >= 0.5:
89        # grid is defined in 1/1000 degrees; old format
90        grid = '{}/{}'.format(float(grid) / 1000.,
91                              float(grid) / 1000.)
92    elif float(grid) / 100. < 0.5:
93        # grid is defined in normal degree; new format
94        grid = '{}/{}'.format(float(grid), float(grid))
95
96    return grid
97
98def check_area(grid, area, upper, lower, left , right):
99    '''Defines the correct area string.
100
101    Checks on the format of the four area components. Wether it is of
102    the order of 1000 or 1. Also checks wether area was already set by command
103    line, then the four components are overwritten.
104    Convert to correct format of the order of magnitude "1" and sets the
105    area parameter (North/West/South/East).
106    E.g.: -5./20./10./10.
107
108    Parameters
109    ----------
110    grid : str
111        Contains grid information.
112
113    area : str
114        Contains area informtion.
115
116    upper : str
117        The northern most latitude.
118
119    lower : str
120        The souther most latitude.
121
122    left : str
123        The western most longitude.
124
125    right : str
126        The eastern most longiude.
127
128    Return
129    ------
130    grid : str
131        Contains grid in format Lat/lon. E.g. 0.1/0.1
132    '''
133    if 'N' in grid:  # Gaussian output grid
134        area = 'G'
135        return area
136
137    # if area was provided decompose area into its 4 components
138    if area:
139        components = area.split('/')
140        upper, left, lower, right = components
141
142    # determine area format
143    if ((abs(float(upper) / 10000.) >= 0.01 or float(upper) / 1000. == 0. ) and
144        (abs(float(lower) / 10000.) >= 0.01 or float(lower) / 1000. == 0. ) and
145        (abs(float(left) / 10000.) >= 0.01 or float(left) / 1000. == 0. ) and
146        (abs(float(right) / 10000.) >= 0.01 or float(right) / 1000. == 0.)):
147        # area is defined in 1/1000 degrees; old format
148        area = '{}/{}/{}/{}'.format(float(upper) / 1000.,
149                                    float(left) / 1000.,
150                                    float(lower) / 1000.,
151                                    float(right) / 1000.)
152    elif (abs(float(upper) / 10000.) < 0.05 and
153          abs(float(lower) / 10000.) < 0.05 and
154          abs(float(left) / 10000.) < 0.05 and
155          abs(float(right) / 10000.) < 0.05):
156        # area is already in new format
157        area = '{}/{}/{}/{}'.format(float(upper),
158                                    float(left),
159                                    float(lower),
160                                    float(right))
161    else:
162        raise ValueError('The area components have different '
163                         'formats (upper, lower, left, right): '
164                         '{}/{}/{}/{}'.format(str(upper), str(lower),
165                                              str(left) , str(right)))
166
167    return area
168
169def check_levels(levelist, level):
170    '''Defines correct level list and guarantees that the maximum level is
171    one of the available maximum levels.
172
173    Parameters
174    ----------
175    levelist : str
176        Specifies the level list.
177        Examples: model level: 1/to/137, pressure levels: 500/to/1000
178
179    level : str
180        Specifies the maximum level.
181
182    Return
183    ------
184    levelist : str
185        Specifies the required levels. It has to have a valid
186        correspondence to the selected levtype.
187        Examples: model level: 1/to/137, pressure levels: 500/to/1000
188
189    level : str
190        Specifies the maximum level. It has to be one of the
191        available maximum level number as contained in the variable
192        MAX_LEVEL_LIST in "_config". E.g. [16, 19, 31, 40, 50, 60, 62, 91, 137]
193
194    '''
195    # assure consistency of levelist and level
196    if not levelist and not level:
197        raise ValueError('ERROR: neither levelist nor level '
198                         'specified in CONTROL file')
199    elif not levelist and level:
200        levelist = '1/to/' + level
201    elif (levelist and not level) or \
202         (levelist[-1] != level[-1]):
203        level = levelist.split('/')[-1]
204    else:
205        pass
206
207    # check if max level is a valid level
208    if int(level) not in _config.MAX_LEVEL_LIST:
209        raise ValueError('ERROR: \n'
210                         'LEVEL must be the maximum level of a specified '
211                         'level list from ECMWF, e.g. {} \n'
212                         'Check parameter "LEVEL" or the max level of '
213                         '"LEVELIST"!'.format(str(_config.MAX_LEVEL_LIST)))
214
215    return levelist, level
216
217
218def check_ppid(c, ppid):
219    '''Sets the current PPID.
220
221    Parameters
222    ----------
223    c : ControlFile
224            Contains all the parameters of CONTROL file and
225            command line.
226
227    ppid : int or None
228        Contains the ppid number provided by the command line parameter
229        of is None otherwise.
230
231    Return
232    ------
233
234    '''
235
236    if not ppid:
237        c.ppid = str(os.getppid())
238    else:
239        c.ppid = ppid
240
241    return
242
243
244def check_purefc(ftype):
245    '''Check for a pure forecast mode.
246
247    Parameters
248    ----------
249    ftype : list of str
250        List of field types.
251
252    Return
253    ------
254    True or False:
255        True if pure forecasts are to be retrieved. False if there are
256        analysis fields in between.
257    '''
258
259    if 'AN' not in ftype and '4V' not in ftype:
260        # pure forecast
261        return 1
262
263    return 0
264
265
266def check_step(step, mailfail):
267    '''Checks on step format and convert into a list of steps.
268
269    If the steps were defined with "to" and "by" they are converted into
270    a list of steps. If the steps were set in a string, it is
271    converted into a list.
272
273    Parameters
274    ----------
275    step : list of str or str
276        Specifies the forecast time step from forecast base time.
277        Valid values are hours (HH) from forecast base time.
278
279    mailfail : list of str
280        Contains all email addresses which should be notified.
281        It might also contain just the ecmwf user name which will trigger
282        mailing to the associated email address for this user.
283
284    Return
285    ------
286    step : list of str
287        List of forecast steps in format e.g. [001, 002, ...]
288    '''
289
290    if '/' in step:
291        steps = step.split('/')
292        if 'to' in step.lower() and 'by' in step.lower():
293            ilist = np.arange(int(steps[0]),
294                              int(steps[2]) + 1,
295                              int(steps[4]))
296            step = ['{:0>3}'.format(i) for i in ilist]
297        elif 'to' in step.lower() and 'by' not in step.lower():
298            my_error(mailfail, step + ':\n' +
299                     'if "to" is used in steps parameter, '
300                     'please use "by" as well')
301        else:
302            step = steps
303
304    if not isinstance(step, list):
305        step = [step]
306
307    return step
308
309def check_type(ftype, steps):
310    '''Check if type variable is of type list and if analysis field has
311    forecast step 0.
312
313    Parameters
314    ----------
315    ftype : list of str or str
316        List of field types.
317
318    steps : str
319        Specifies the forecast time step from forecast base time.
320        Valid values are hours (HH) from forecast base time.
321
322    Return
323    ------
324    ftype : list of str
325        List of field types.
326    '''
327    if not isinstance(ftype, list):
328        ftype = [ftype]
329
330    for i, val in enumerate(ftype):
331        if ftype[i] == 'AN' and int(steps[i]) != 0:
332            print('Analysis retrievals must have STEP = 0 (now set to 0)')
333            ftype[i] = 0
334
335    return ftype
336
337def check_time(ftime):
338    '''Check if time variable is of type list. Otherwise convert to list.
339
340    Parameters
341    ----------
342    ftime : list of str or str
343        The time in hours of the field.
344
345    Return
346    ------
347    ftime : list of str
348        The time in hours of the field.
349    '''
350    if not isinstance(ftime, list):
351        ftime = [ftime]
352
353    return ftime
354
355def check_len_type_time_step(ftype, ftime, steps, maxstep, purefc):
356    '''Check if
357
358    Parameters
359    ----------
360    ftype : list of str
361        List of field types.
362
363    ftime : list of str or str
364        The time in hours of the field.
365
366    steps : str
367        Specifies the forecast time step from forecast base time.
368        Valid values are hours (HH) from forecast base time.
369
370    maxstep : int
371        The maximum forecast time step in hours from the forecast base time.
372        This is the maximum step for non flux (accumulated) forecast data.
373
374    purefc : int
375        Switch for definition of pure forecast mode or not.
376
377    Return
378    ------
379    ftype : list of str
380        List of field types.
381
382    ftime : list of str
383        The time in hours of the field.
384
385    steps : str
386        Specifies the forecast time step from forecast base time.
387        Valid values are hours (HH) from forecast base time.
388    '''
389    if not (len(ftype) == len(ftime) == len(steps)):
390        raise ValueError('ERROR: The number of field types, times and steps '
391                         'are not the same! Please check the setting in the '
392                         'CONTROL file!')
393
394    # if pure forecast is selected and only one field type/time is set
395    # prepare a complete list of type/time/step combination upto maxstep
396    if len(ftype) == 1 and purefc:
397        nftype = []
398        nsteps = []
399        nftime = []
400        for i in range(0, maxstep + 1):
401            nftype.append(ftype[0])
402            nsteps.append('{:0>3}'.format(i))
403            nftime.append(ftime[0])
404        return nftype, nftime, nsteps
405
406    return ftype, ftime, steps
407
408def check_mail(mail):
409    '''Check the string of mail addresses, seperate them and convert to a list.
410
411    Parameters
412    ----------
413    mail : list of str or str
414        Contains email addresses for notifications.
415        It might also contain just the ecmwf user name which will trigger
416        mailing to the associated email address for this user.
417
418    Return
419    ------
420    mail : list of str
421        Contains email addresses for notifications.
422        It might also contain just the ecmwf user name which will trigger
423        mailing to the associated email address for this user.
424
425    '''
426    if not isinstance(mail, list):
427        if ',' in mail:
428            mail = mail.split(',')
429        elif ' ' in mail:
430            mail = mail.split()
431        else:
432            mail = [mail]
433
434    return mail
435
436def check_queue(queue, gateway, destination, ecuid, ecgid):
437    '''Check if the necessary ECMWF parameters are set if the queue is
438    one of the QUEUES_LIST (in _config).
439
440    Parameters
441    ----------
442    queue : str
443        Name of the queue if submitted to the ECMWF servers.
444        Used to check if ecuid, ecgid, gateway and destination
445        are set correctly and are not empty.
446
447    gateway : str
448        The address of the gateway server.
449
450    destination : str
451        The name of the destination of the gateway server for data
452        transfer through ectrans. E.g. name@genericSftp
453
454    ecuid : str
455        ECMWF user id.
456
457    ecgid : str
458        ECMWF group id.
459
460    Return
461    ------
462
463    '''
464    if queue in _config.QUEUES_LIST and \
465        not gateway or not destination or \
466        not ecuid or not ecgid:
467        raise ValueError('\nEnvironment variables GATEWAY, DESTINATION, ECUID '
468                         'and ECGID were not set properly! \n '
469                         'Please check for existence of file "ECMWF_ENV" '
470                         'in the run directory!')
471    return
472
473def check_pathes(idir, odir, fpdir, fedir):
474    '''Check if output and flexpart pathes are set.
475
476    Parameters
477    ----------
478    idir : str
479        Path to the temporary directory for MARS retrieval data.
480
481    odir : str
482        Path to the final output directory where the FLEXPART input files
483        will be stored.
484
485    fpdir : str
486        Path to FLEXPART root directory.
487
488    fedir : str
489        Path to flex_extract root directory.
490
491    Return
492    ------
493    odir : str
494        Path to the final output directory where the FLEXPART input files
495        will be stored.
496
497    fpdir : str
498        Path to FLEXPART root directory.
499
500    '''
501    if not fpdir:
502        fpdir = fedir
503
504    if not odir:
505        odir = idir
506
507    return odir, fpdir
508
509def check_dates(start, end):
510    '''Checks if there is at least a start date for a one day retrieval.
511
512    Checks if end date lies after start date and end date is set.
513
514    Parameters
515    ----------
516    start : str
517        The start date of the retrieval job.
518
519    end : str
520        The end date of the retrieval job.
521
522    Return
523    ------
524    start : str
525        The start date of the retrieval job.
526
527    end : str
528        The end date of the retrieval job.
529
530    '''
531    # check for having at least a starting date
532    # otherwise program is not allowed to run
533    if not start:
534        raise ValueError('start_date was neither specified in command line nor '
535                         'in CONTROL file.\n'
536                         'Try "{} -h" to print usage information'
537                         .format(sys.argv[0].split('/')[-1]) )
538
539    # retrieve just one day if end_date isn't set
540    if not end:
541        end = start
542
543    dstart = datetime.strptime(start, '%Y%m%d')
544    dend = datetime.strptime(end, '%Y%m%d')
545    if dstart > dend:
546        raise ValueError('ERROR: Start date is after end date! \n'
547                         'Please adapt the dates in CONTROL file or '
548                         'command line! (start={}; end={})'.format(start, end))
549
550    return start, end
551
552def check_maxstep(maxstep, steps):
553    '''Convert maxstep into integer if it is already given. Otherwise, select
554    maxstep by going through the steps list.
555
556    Parameters
557    ----------
558    maxstep : str
559        The maximum forecast time step in hours from the forecast base time.
560        This is the maximum step for non flux (accumulated) forecast data.
561
562    steps : str
563        Specifies the forecast time step from forecast base time.
564        Valid values are hours (HH) from forecast base time.
565
566    Return
567    ------
568    maxstep : int
569        The maximum forecast time step in hours from the forecast base time.
570        This is the maximum step for non flux (accumulated) forecast data.
571
572    '''
573    # if maxstep wasn't provided
574    # search for it in the "step" parameter
575    if not maxstep:
576        maxstep = 0
577        for s in steps:
578            if int(s) > maxstep:
579                maxstep = int(s)
580    else:
581        maxstep = int(maxstep)
582
583    return maxstep
584
585def check_basetime(basetime):
586    '''Check if basetime is set and contains one of the two
587    possible values (0, 12).
588
589    Parameters
590    ----------
591    basetime : int or str or None
592        The time for a half day retrieval. The 12 hours upfront are to be
593        retrieved.
594
595    Return
596    ------
597    basetime : int or None
598        The time for a half day retrieval. The 12 hours upfront are to be
599        retrieved.
600    '''
601    if basetime is not None:
602        basetime = int(basetime)
603        if basetime != 0 and basetime != 12:
604            raise ValueError('ERROR: Basetime has an invalid value '
605                             '-> {}'.format(str(basetime)))
606    return basetime
607
608def check_request(request, marsfile):
609    '''Check if there is an old mars request file and remove it.
610
611    Parameters
612    ----------
613    request : int
614        Selects the mode of retrieval.
615        0: Retrieves the data from ECMWF.
616        1: Prints the mars requests to an output file.
617        2: Retrieves the data and prints the mars request.
618
619    marsfile : str
620        Path to the mars request file.
621
622    Return
623    ------
624
625    '''
626    if request != 0:
627        if os.path.isfile(marsfile):
628            silent_remove(marsfile)
629    return
630
631def check_public(public, dataset):
632    '''Check wether the dataset parameter is set for a
633    public data set retrieval.
634
635    Parameters
636    ----------
637    public : int
638        Specifies if public data are to be retrieved or not.
639
640    dataset : str
641        Specific name which identifies the public dataset.
642
643    Return
644    ------
645
646    '''
647    if public and not dataset:
648        raise ValueError('ERROR: If public mars data wants to be retrieved, '
649                         'the "dataset"-parameter has to be set too!')
650    return
651
652def check_acctype(acctype, ftype):
653    '''Guarantees that the accumulation field type is set.
654
655    If not set, it is derivated as in the old method (TYPE[1]).
656
657    Parameters
658    ----------
659    acctype : str
660        The field type for the accumulated forecast fields.
661
662    ftype : list of str
663        List of field types.
664
665    Return
666    ------
667    acctype : str
668        The field type for the accumulated forecast fields.
669    '''
670    if not acctype:
671        print('... Control parameter ACCTYPE was not defined.')
672        try:
673            if len(ftype) == 1 and ftype[0] != 'AN':
674                print('... Use same field type as for the non-flux fields.')
675                acctype = ftype[0]
676            elif len(ftype) > 1 and ftype[1] != 'AN':
677                print('... Use old setting by using TYPE[1] for flux forecast!')
678                acctype = ftype[1]
679        except:
680            raise ValueError('ERROR: Accumulation field type could not be set!')
681    else:
682        if acctype.upper() == 'AN':
683            raise ValueError('ERROR: Accumulation forecast fields can not be '
684                             'of type "analysis"!')
685    return acctype
686
687
688def check_acctime(acctime, marsclass, purefc):
689    '''Guarantees that the accumulation forecast times were set.
690
691    If it is not set, it tries to set the value for some of the
692    most commonly used data sets. Otherwise it raises an error.
693
694    Parameters
695    ----------
696    acctime : str
697        The starting time from the accumulated forecasts.
698
699    marsclass : str
700        ECMWF data classification identifier.
701
702    purefc : int
703        Switch for definition of pure forecast mode or not.
704
705    Return
706    ------
707    acctime : str
708        The starting time from the accumulated forecasts.
709    '''
710
711    if not acctime:
712        print('... Control parameter ACCTIME was not defined.')
713        print('... Value will be set depending on field type:\n '
714              '\t\t EA=06/18\n\t\t EI/OD=00/12\n\t\t EP=18')
715        if marsclass.upper() == 'EA': # Era 5
716            acctime = '06/18'
717        elif marsclass.upper() == 'EI': # Era-Interim
718            acctime = '00/12'
719        elif marsclass.upper() == 'EP': # CERA
720            acctime = '18'
721        elif marsclass.upper() == 'OD' and not purefc: # On-demand
722            acctime = '00/12'
723        else:
724            raise ValueError('ERROR: Accumulation forecast time can not '
725                             'automatically be derived!')
726    return acctime
727
728def check_accmaxstep(accmaxstep, marsclass, purefc, maxstep):
729    '''Guarantees that the accumulation forecast step were set.
730
731    Parameters
732    ----------
733    accmaxstep : str
734        The maximum forecast step for the accumulated forecast fields.
735
736    marsclass : str
737        ECMWF data classification identifier.
738
739    purefc : int
740        Switch for definition of pure forecast mode or not.
741
742    maxstep : str
743        The maximum forecast time step in hours from the forecast base time.
744        This is the maximum step for non flux (accumulated) forecast data.
745
746    Return
747    ------
748    accmaxstep : str
749        The maximum forecast step for the accumulated forecast fields.
750    '''
751    if not accmaxstep:
752        print('... Control parameter ACCMAXSTEP was not defined.')
753        print('... Value will be set depending on field type/time: '
754              '\n\t\t EA/EI/OD=12\n\t\t EP=24')
755        if marsclass.upper() in ['EA', 'EI', 'OD'] and not purefc:
756            # Era 5, Era-Interim, On-demand operational
757            accmaxstep = '12'
758        elif marsclass.upper() == 'EP': # CERA
759            accmaxstep = '24'
760        elif purefc and accmaxstep != maxstep:
761            accmaxstep = maxstep
762            print('... For pure forecast mode, the accumulated forecast must '
763                  'have the same maxstep as the normal forecast fields!\n'
764                  '\t\t Accmaxstep was set to maxstep!')
765        else:
766            raise ValueError('ERROR: Accumulation forecast step can not '
767                             'automatically be derived!')
768    else:
769        if purefc and int(accmaxstep) != int(maxstep):
770            accmaxstep = maxstep
771            print('... For pure forecast mode, the accumulated forecast must '
772                          'have the same maxstep as the normal forecast fields!\n'
773                          '\t\t Accmaxstep was set to maxstep!')
774    return accmaxstep
775
776def check_addpar(addpar):
777    '''Check that addpar has correct format of additional parameters in
778    a single string, so that it can be easily appended to the hard coded
779    parameters that are retrieved in any case.
780
781    Parameters
782    ----------
783    addpar : str or list of str
784        List of additional parameters to be retrieved.
785
786    Return
787    ------
788    addpar : str
789        List of additional parameters to be retrieved.
790    '''
791
792    if addpar and isinstance(addpar, str):
793        if '/' in addpar:
794            parlist = addpar.split('/')
795            parlist = [p for p in parlist if p is not '']
796        else:
797            parlist = [addpar]
798
799        addpar = '/' + '/'.join(parlist)
800
801    return addpar
802
803
804def check_job_chunk(job_chunk):
805    '''Checks that if job chunk is set, the number is positive and non zero.
806
807    Parameters
808    ----------
809    job_chunk : int
810        The number of days for a single job script.
811
812    Return
813    ------
814    job_chunk : int
815        The number of days for a single job script.
816    '''
817    if not job_chunk:
818        return job_chunk
819
820    if job_chunk < 0:
821        raise ValueError('ERROR: The number of job chunk is negative!\n'
822                         'It has to be a positive number!')
823    elif job_chunk == 0:
824        job_chunk = None
825    else:
826        pass
827
828    return job_chunk
829
830
831def check_number(number, mailfail):
832    '''Check for correct string format of ensemble member numbers.
833
834    Parameters
835    ----------
836    number : str
837        List of ensemble member forecast runs.
838
839    mailfail : list of str
840        Contains all email addresses which should be notified.
841        It might also contain just the ecmwf user name which will trigger
842        mailing to the associated email address for this user.
843
844    Return
845    ------
846    number : str
847        String with list of ensemble member forecast runs. E.g. '01/02/03/04'
848    '''
849
850    if '/' in number:
851        numbers = number.split('/')
852        if 'to' in number.lower() and 'by' in number.lower():
853            number = '{:0>3}'.format(int(numbers[0])) + '/TO/' + \
854                     '{:0>3}'.format(int(numbers[2])) + '/BY/' + \
855                     '{:0>3}'.format(int(numbers[4]))
856        elif 'to' in number.lower() and 'by' not in number.lower():
857            number = '{:0>3}'.format(int(numbers[0])) + '/TO/' + \
858                     '{:0>3}'.format(int(numbers[2]))
859        else:
860            numbers = ['{:0>3}'.format(i) for i in numbers]
861            number = '{:0>3}/'.join(numbers)
862    elif number.isdigit():
863        number = '{:0>3}'.format(int(number))
864    else:
865        pass
866
867    return number
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG