#!/usr/bin/env python # -*- coding: utf-8 -*- #******************************************************************************* # @Author: Anne Philipp (University of Vienna) # # @Date: November 2018 # # @Change History: # # @License: # (C) Copyright 2014-2019. # Anne Philipp, Leopold Haimberger # # This work is licensed under the Creative Commons Attribution 4.0 # International License. To view a copy of this license, visit # http://creativecommons.org/licenses/by/4.0/ or send a letter to # Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. #******************************************************************************* '''This module contains check methods for the CONTROL paramaters. ''' # ------------------------------------------------------------------------------ # MODULES # ------------------------------------------------------------------------------ import os import sys import _config import exceptions from tools import my_error, silent_remove from datetime import datetime import numpy as np # ------------------------------------------------------------------------------ # FUNCTIONS # ------------------------------------------------------------------------------ def check_logicals_type(c, logicals): '''Check that the logical variables have correct type integer. Parameters ---------- c : ControlFile Contains all the parameters of CONTROL file and command line. logicals : list of (str or int) Names of the switches that are used to control the flow of the program. Return ------ ''' for var in logicals: if not isinstance(getattr(c, var), int): setattr(c, var, int(getattr(c, var))) return def check_grid(grid): '''Convert grid into correct Lat/Lon format. E.g. '0.5/0.5' Checks on format of original grid. Wether it is in the order of 1000 or 1. Convert to correct grid format and substitute into "Lat/Lon" format string. Parameters ---------- grid : str Contains grid information Return ------ grid : str Contains grid in format Lat/lon. E.g. 0.1/0.1 ''' if 'N' in grid: return grid if '/' in grid: gridx, gridy = grid.split('/') if gridx == gridy: grid = gridx else: raise ValueError('GRID parameter contains two values ' 'which are unequal %s' (grid)) # determine grid format if float(grid) / 100. >= 0.5: # grid is defined in 1/1000 degrees; old format grid = '{}/{}'.format(float(grid) / 1000., float(grid) / 1000.) elif float(grid) / 100. < 0.5: # grid is defined in normal degree; new format grid = '{}/{}'.format(float(grid), float(grid)) return grid def check_area(grid, area, upper, lower, left , right): '''Defines the correct area string. Checks on the format of the four area components. Wether it is of the order of 1000 or 1. Also checks wether area was already set by command line, then the four components are overwritten. Convert to correct format of the order of magnitude "1" and sets the area parameter (North/West/South/East). E.g.: -5./20./10./10. Parameters ---------- grid : str Contains grid information. area : str Contains area informtion. upper : str The northern most latitude. lower : str The souther most latitude. left : str The western most longitude. right : str The eastern most longiude. Return ------ grid : str Contains grid in format Lat/lon. E.g. 0.1/0.1 ''' if 'N' in grid: # Gaussian output grid area = 'G' return area # if area was provided decompose area into its 4 components if area: components = area.split('/') upper, left, lower, right = components # determine area format if ((abs(float(upper) / 10000.) >= 0.01 or float(upper) / 1000. == 0. ) and (abs(float(lower) / 10000.) >= 0.01 or float(lower) / 1000. == 0. ) and (abs(float(left) / 10000.) >= 0.01 or float(left) / 1000. == 0. ) and (abs(float(right) / 10000.) >= 0.01 or float(right) / 1000. == 0.)): # area is defined in 1/1000 degrees; old format area = '{}/{}/{}/{}'.format(float(upper) / 1000., float(left) / 1000., float(lower) / 1000., float(right) / 1000.) elif (abs(float(upper) / 10000.) < 0.05 and abs(float(lower) / 10000.) < 0.05 and abs(float(left) / 10000.) < 0.05 and abs(float(right) / 10000.) < 0.05): # area is already in new format area = '{}/{}/{}/{}'.format(float(upper), float(left), float(lower), float(right)) else: raise ValueError('The area components have different ' 'formats (upper, lower, left, right): ' '{}/{}/{}/{}'.format(str(upper), str(lower), str(left) , str(right))) return area def check_levels(levelist, level): '''Defines correct level list and guarantees that the maximum level is one of the available maximum levels. Parameters ---------- levelist : str Specifies the level list. Examples: model level: 1/to/137, pressure levels: 500/to/1000 level : str Specifies the maximum level. Return ------ levelist : str Specifies the required levels. It has to have a valid correspondence to the selected levtype. Examples: model level: 1/to/137, pressure levels: 500/to/1000 level : str Specifies the maximum level. It has to be one of the available maximum level number as contained in the variable MAX_LEVEL_LIST in "_config". E.g. [16, 19, 31, 40, 50, 60, 62, 91, 137] ''' # assure consistency of levelist and level if not levelist and not level: raise ValueError('ERROR: neither levelist nor level ' 'specified in CONTROL file') elif not levelist and level: levelist = '1/to/' + level elif (levelist and not level) or \ (levelist[-1] != level[-1]): level = levelist.split('/')[-1] else: pass # check if max level is a valid level if int(level) not in _config.MAX_LEVEL_LIST: raise ValueError('ERROR: \n' 'LEVEL must be the maximum level of a specified ' 'level list from ECMWF, e.g. {} \n' 'Check parameter "LEVEL" or the max level of ' '"LEVELIST"!'.format(str(_config.MAX_LEVEL_LIST))) return levelist, level def check_ppid(c, ppid): '''Sets the current PPID. Parameters ---------- c : ControlFile Contains all the parameters of CONTROL file and command line. ppid : int or None Contains the ppid number provided by the command line parameter of is None otherwise. Return ------ ''' if not ppid: c.ppid = str(os.getppid()) else: c.ppid = ppid return def check_purefc(ftype): '''Check for a pure forecast mode. Parameters ---------- ftype : list of str List of field types. Return ------ True or False: True if pure forecasts are to be retrieved. False if there are analysis fields in between. ''' if 'AN' not in ftype and '4V' not in ftype: # pure forecast return 1 return 0 def check_step(step, mailfail): '''Checks on step format and convert into a list of steps. If the steps were defined with "to" and "by" they are converted into a list of steps. If the steps were set in a string, it is converted into a list. Parameters ---------- step : list of str or str Specifies the forecast time step from forecast base time. Valid values are hours (HH) from forecast base time. mailfail : list of str Contains all email addresses which should be notified. It might also contain just the ecmwf user name which will trigger mailing to the associated email address for this user. Return ------ step : list of str List of forecast steps in format e.g. [001, 002, ...] ''' if '/' in step: steps = step.split('/') if 'to' in step.lower() and 'by' in step.lower(): ilist = np.arange(int(steps[0]), int(steps[2]) + 1, int(steps[4])) step = ['{:0>3}'.format(i) for i in ilist] elif 'to' in step.lower() and 'by' not in step.lower(): my_error(mailfail, step + ':\n' + 'if "to" is used in steps parameter, ' 'please use "by" as well') else: step = steps if not isinstance(step, list): step = [step] return step def check_type(ftype, steps): '''Check if type variable is of type list and if analysis field has forecast step 0. Parameters ---------- ftype : list of str or str List of field types. steps : str Specifies the forecast time step from forecast base time. Valid values are hours (HH) from forecast base time. Return ------ ftype : list of str List of field types. ''' if not isinstance(ftype, list): ftype = [ftype] for i, val in enumerate(ftype): if ftype[i] == 'AN' and int(steps[i]) != 0: print('Analysis retrievals must have STEP = 0 (now set to 0)') ftype[i] = 0 return ftype def check_time(ftime): '''Check if time variable is of type list. Otherwise convert to list. Parameters ---------- ftime : list of str or str The time in hours of the field. Return ------ ftime : list of str The time in hours of the field. ''' if not isinstance(ftime, list): ftime = [ftime] return ftime def check_len_type_time_step(ftype, ftime, steps, maxstep, purefc): '''Check if Parameters ---------- ftype : list of str List of field types. ftime : list of str or str The time in hours of the field. steps : str Specifies the forecast time step from forecast base time. Valid values are hours (HH) from forecast base time. maxstep : int The maximum forecast time step in hours from the forecast base time. This is the maximum step for non flux (accumulated) forecast data. purefc : int Switch for definition of pure forecast mode or not. Return ------ ftype : list of str List of field types. ftime : list of str The time in hours of the field. steps : str Specifies the forecast time step from forecast base time. Valid values are hours (HH) from forecast base time. ''' if not (len(ftype) == len(ftime) == len(steps)): raise ValueError('ERROR: The number of field types, times and steps ' 'are not the same! Please check the setting in the ' 'CONTROL file!') # if pure forecast is selected and only one field type/time is set # prepare a complete list of type/time/step combination upto maxstep if len(ftype) == 1 and purefc: nftype = [] nsteps = [] nftime = [] for i in range(0, maxstep + 1): nftype.append(ftype[0]) nsteps.append('{:0>3}'.format(i)) nftime.append(ftime[0]) return nftype, nftime, nsteps return ftype, ftime, steps def check_mail(mail): '''Check the string of mail addresses, seperate them and convert to a list. Parameters ---------- mail : list of str or str Contains email addresses for notifications. It might also contain just the ecmwf user name which will trigger mailing to the associated email address for this user. Return ------ mail : list of str Contains email addresses for notifications. It might also contain just the ecmwf user name which will trigger mailing to the associated email address for this user. ''' if not isinstance(mail, list): if ',' in mail: mail = mail.split(',') elif ' ' in mail: mail = mail.split() else: mail = [mail] return mail def check_queue(queue, gateway, destination, ecuid, ecgid): '''Check if the necessary ECMWF parameters are set if the queue is one of the QUEUES_LIST (in _config). Parameters ---------- queue : str Name of the queue if submitted to the ECMWF servers. Used to check if ecuid, ecgid, gateway and destination are set correctly and are not empty. gateway : str The address of the gateway server. destination : str The name of the destination of the gateway server for data transfer through ectrans. E.g. name@genericSftp ecuid : str ECMWF user id. ecgid : str ECMWF group id. Return ------ ''' if queue in _config.QUEUES_LIST and \ not gateway or not destination or \ not ecuid or not ecgid: raise ValueError('\nEnvironment variables GATEWAY, DESTINATION, ECUID ' 'and ECGID were not set properly! \n ' 'Please check for existence of file "ECMWF_ENV" ' 'in the run directory!') return def check_pathes(idir, odir, fpdir, fedir): '''Check if output and flexpart pathes are set. Parameters ---------- idir : str Path to the temporary directory for MARS retrieval data. odir : str Path to the final output directory where the FLEXPART input files will be stored. fpdir : str Path to FLEXPART root directory. fedir : str Path to flex_extract root directory. Return ------ odir : str Path to the final output directory where the FLEXPART input files will be stored. fpdir : str Path to FLEXPART root directory. ''' if not fpdir: fpdir = fedir if not odir: odir = idir return odir, fpdir def check_dates(start, end): '''Checks if there is at least a start date for a one day retrieval. Checks if end date lies after start date and end date is set. Parameters ---------- start : str The start date of the retrieval job. end : str The end date of the retrieval job. Return ------ start : str The start date of the retrieval job. end : str The end date of the retrieval job. ''' # check for having at least a starting date # otherwise program is not allowed to run if not start: raise ValueError('start_date was neither specified in command line nor ' 'in CONTROL file.\n' 'Try "{} -h" to print usage information' .format(sys.argv[0].split('/')[-1]) ) # retrieve just one day if end_date isn't set if not end: end = start dstart = datetime.strptime(start, '%Y%m%d') dend = datetime.strptime(end, '%Y%m%d') if dstart > dend: raise ValueError('ERROR: Start date is after end date! \n' 'Please adapt the dates in CONTROL file or ' 'command line! (start={}; end={})'.format(start, end)) return start, end def check_maxstep(maxstep, steps): '''Convert maxstep into integer if it is already given. Otherwise, select maxstep by going through the steps list. Parameters ---------- maxstep : str The maximum forecast time step in hours from the forecast base time. This is the maximum step for non flux (accumulated) forecast data. steps : str Specifies the forecast time step from forecast base time. Valid values are hours (HH) from forecast base time. Return ------ maxstep : int The maximum forecast time step in hours from the forecast base time. This is the maximum step for non flux (accumulated) forecast data. ''' # if maxstep wasn't provided # search for it in the "step" parameter if not maxstep: maxstep = 0 for s in steps: if int(s) > maxstep: maxstep = int(s) else: maxstep = int(maxstep) return maxstep def check_basetime(basetime): '''Check if basetime is set and contains one of the two possible values (0, 12). Parameters ---------- basetime : int or str or None The time for a half day retrieval. The 12 hours upfront are to be retrieved. Return ------ basetime : int or None The time for a half day retrieval. The 12 hours upfront are to be retrieved. ''' if basetime is not None: basetime = int(basetime) if basetime != 0 and basetime != 12: raise ValueError('ERROR: Basetime has an invalid value ' '-> {}'.format(str(basetime))) return basetime def check_request(request, marsfile): '''Check if there is an old mars request file and remove it. Parameters ---------- request : int Selects the mode of retrieval. 0: Retrieves the data from ECMWF. 1: Prints the mars requests to an output file. 2: Retrieves the data and prints the mars request. marsfile : str Path to the mars request file. Return ------ ''' if request != 0: if os.path.isfile(marsfile): silent_remove(marsfile) return def check_public(public, dataset): '''Check wether the dataset parameter is set for a public data set retrieval. Parameters ---------- public : int Specifies if public data are to be retrieved or not. dataset : str Specific name which identifies the public dataset. Return ------ ''' if public and not dataset: raise ValueError('ERROR: If public mars data wants to be retrieved, ' 'the "dataset"-parameter has to be set too!') return def check_acctype(acctype, ftype): '''Guarantees that the accumulation field type is set. If not set, it is derivated as in the old method (TYPE[1]). Parameters ---------- acctype : str The field type for the accumulated forecast fields. ftype : list of str List of field types. Return ------ acctype : str The field type for the accumulated forecast fields. ''' if not acctype: print('... Control parameter ACCTYPE was not defined.') try: if len(ftype) == 1 and ftype[0] != 'AN': print('... Use same field type as for the non-flux fields.') acctype = ftype[0] elif len(ftype) > 1 and ftype[1] != 'AN': print('... Use old setting by using TYPE[1] for flux forecast!') acctype = ftype[1] except: raise ValueError('ERROR: Accumulation field type could not be set!') else: if acctype.upper() == 'AN': raise ValueError('ERROR: Accumulation forecast fields can not be ' 'of type "analysis"!') return acctype def check_acctime(acctime, marsclass, purefc): '''Guarantees that the accumulation forecast times were set. If it is not set, it tries to set the value for some of the most commonly used data sets. Otherwise it raises an error. Parameters ---------- acctime : str The starting time from the accumulated forecasts. marsclass : str ECMWF data classification identifier. purefc : int Switch for definition of pure forecast mode or not. Return ------ acctime : str The starting time from the accumulated forecasts. ''' if not acctime: print('... Control parameter ACCTIME was not defined.') print('... Value will be set depending on field type:\n ' '\t\t EA=06/18\n\t\t EI/OD=00/12\n\t\t EP=18') if marsclass.upper() == 'EA': # Era 5 acctime = '06/18' elif marsclass.upper() == 'EI': # Era-Interim acctime = '00/12' elif marsclass.upper() == 'EP': # CERA acctime = '18' elif marsclass.upper() == 'OD' and not purefc: # On-demand acctime = '00/12' else: raise ValueError('ERROR: Accumulation forecast time can not ' 'automatically be derived!') return acctime def check_accmaxstep(accmaxstep, marsclass, purefc, maxstep): '''Guarantees that the accumulation forecast step were set. Parameters ---------- accmaxstep : str The maximum forecast step for the accumulated forecast fields. marsclass : str ECMWF data classification identifier. purefc : int Switch for definition of pure forecast mode or not. maxstep : str The maximum forecast time step in hours from the forecast base time. This is the maximum step for non flux (accumulated) forecast data. Return ------ accmaxstep : str The maximum forecast step for the accumulated forecast fields. ''' if not accmaxstep: print('... Control parameter ACCMAXSTEP was not defined.') print('... Value will be set depending on field type/time: ' '\n\t\t EA/EI/OD=12\n\t\t EP=24') if marsclass.upper() in ['EA', 'EI', 'OD'] and not purefc: # Era 5, Era-Interim, On-demand operational accmaxstep = '12' elif marsclass.upper() == 'EP': # CERA accmaxstep = '24' elif purefc and accmaxstep != maxstep: accmaxstep = maxstep print('... For pure forecast mode, the accumulated forecast must ' 'have the same maxstep as the normal forecast fields!\n' '\t\t Accmaxstep was set to maxstep!') else: raise ValueError('ERROR: Accumulation forecast step can not ' 'automatically be derived!') else: if purefc and int(accmaxstep) != int(maxstep): accmaxstep = maxstep print('... For pure forecast mode, the accumulated forecast must ' 'have the same maxstep as the normal forecast fields!\n' '\t\t Accmaxstep was set to maxstep!') return accmaxstep def check_addpar(addpar): '''Check that addpar has correct format of additional parameters in a single string, so that it can be easily appended to the hard coded parameters that are retrieved in any case. Parameters ---------- addpar : str or list of str List of additional parameters to be retrieved. Return ------ addpar : str List of additional parameters to be retrieved. ''' if addpar and isinstance(addpar, str): if '/' in addpar: parlist = addpar.split('/') parlist = [p for p in parlist if p is not ''] else: parlist = [addpar] addpar = '/' + '/'.join(parlist) return addpar def check_job_chunk(job_chunk): '''Checks that if job chunk is set, the number is positive and non zero. Parameters ---------- job_chunk : int The number of days for a single job script. Return ------ job_chunk : int The number of days for a single job script. ''' if not job_chunk: return job_chunk if job_chunk < 0: raise ValueError('ERROR: The number of job chunk is negative!\n' 'It has to be a positive number!') elif job_chunk == 0: job_chunk = None else: pass return job_chunk def check_number(number, mailfail): '''Check for correct string format of ensemble member numbers. Parameters ---------- number : str List of ensemble member forecast runs. mailfail : list of str Contains all email addresses which should be notified. It might also contain just the ecmwf user name which will trigger mailing to the associated email address for this user. Return ------ number : str String with list of ensemble member forecast runs. E.g. '01/02/03/04' ''' if '/' in number: numbers = number.split('/') if 'to' in number.lower() and 'by' in number.lower(): number = '{:0>3}'.format(int(numbers[0])) + '/TO/' + \ '{:0>3}'.format(int(numbers[2])) + '/BY/' + \ '{:0>3}'.format(int(numbers[4])) elif 'to' in number.lower() and 'by' not in number.lower(): number = '{:0>3}'.format(int(numbers[0])) + '/TO/' + \ '{:0>3}'.format(int(numbers[2])) else: numbers = ['{:0>3}'.format(i) for i in numbers] number = '{:0>3}/'.join(numbers) elif number.isdigit(): number = '{:0>3}'.format(int(number)) else: pass return number