Context Navigation

source: flex_extract.git/source/python/classes/ControlFile.py @ 2d56c04

ctbtodev

Last change on this file since 2d56c04 was 2d56c04, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
automatic detection of grid and area component formats (1/1000 or normal degree format)
Property mode set to `100644`
File size: 20.6 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - applied some minor modifications in programming style/structure
14	# - changed name of class Control to ControlFile for more
15	# self-explanation naming
16	# - outsource of class ControlFile
17	# - initialisation of class attributes ( to avoid high number of
18	# conditional statements and set default values )
19	# - divided assignment of attributes and the check of conditions
20	# - outsourced the commandline argument assignments to control attributes
21	#
22	# @License:
23	# (C) Copyright 2015-2018.
24	#
25	# This software is licensed under the terms of the Apache Licence Version 2.0
26	# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
27	#
28	# @Class Description:
29	# The CONTROL file is the steering part of the FLEXPART extraction
30	# software. All necessary parameters needed to retrieve the data fields
31	# from the MARS archive for driving FLEXPART are set in a CONTROL file.
32	# Some specific parameters like the start and end dates can be overwritten
33	# by the command line parameters, but in generel all parameters needed
34	# for a complete set of fields for FLEXPART can be set in the CONTROL file.
35	#
36	# @Class Content:
37	# - __init__
38	# - __read_controlfile__
39	# - __str__
40	# - assign_args_to_control
41	# - assign_envs_to_control
42	# - check_conditions
43	# - check_install_conditions
44	# - to_list
45	#
46	# @Class Attributes:
47	#
48	#
49	#*******************************************************************************
50
51	# ------------------------------------------------------------------------------
52	# MODULES
53	# ------------------------------------------------------------------------------
54	import os
55	import re
56	import sys
57	import inspect
58
59	# software specific classes and modules from flex_extract
60	sys.path.append('../')
61	import _config
62	from mods.tools import my_error, silent_remove
63
64	# ------------------------------------------------------------------------------
65	# CLASS
66	# ------------------------------------------------------------------------------
67	class ControlFile(object):
68	'''
69	Contains the information which are stored in the CONTROL files.
70	'''
71
72	def __init__(self, filename):
73	'''Initialises the instance of ControlFile class and defines
74	all class attributes with default values. Afterwards calls
75	function __read_controlfile__ to read parameter from Control file.
76
77	Parameters
78	----------
79	filename : :obj:`string`
80	Name of CONTROL file.
81
82	Return
83	------
84
85	'''
86
87	# list of all possible class attributes and their default values
88	self.controlfile = filename
89	self.start_date = None
90	self.end_date = None
91	self.date_chunk = 3
92	self.dtime = None
93	self.basetime = None
94	self.maxstep = None
95	self.type = None
96	self.time = None
97	self.step = None
98	self.acctype = None
99	self.acctime = None
100	self.accmaxstep = None
101	self.marsclass = None
102	self.dataset = None
103	self.stream = None
104	self.number = 'OFF'
105	self.expver = '1'
106	self.gaussian = ''
107	self.grid = None
108	self.area = ''
109	self.left = None
110	self.lower = None
111	self.upper = None
112	self.right = None
113	self.level = None
114	self.levelist = None
115	self.resol = None
116	self.gauss = 0
117	self.accuracy = 24
118	self.omega = 0
119	self.omegadiff = 0
120	self.eta = 0
121	self.etadiff = 0
122	self.etapar = 77
123	self.dpdeta = 1
124	self.smooth = 0
125	self.format = 'GRIB1'
126	self.addpar = None
127	self.prefix = 'EN'
128	self.cwc = 0
129	self.wrf = 0
130	self.ecfsdir = 'ectmp:/${USER}/econdemand/'
131	self.mailfail = ['${USER}']
132	self.mailops = ['${USER}']
133	self.grib2flexpart = 0
134	self.ecstorage = 0
135	self.ectrans = 0
136	self.inputdir = _config.PATH_INPUT_DIR
137	self.outputdir = None
138	self.ecmwfdatadir = _config.PATH_FLEXEXTRACT_DIR
139	self.exedir = _config.PATH_FORTRAN_SRC
140	self.flexpart_root_scripts = None
141	self.makefile = 'Makefile.gfortran'
142	self.destination = None
143	self.gateway = None
144	self.ecuid = None
145	self.ecgid = None
146	self.install_target = None
147	self.debug = 0
148	self.request = 0
149	self.public = 0
150	self.ecapi = None
151
152	self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
153	'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
154	'ectrans', 'debug', 'request', 'public']
155
156	self.__read_controlfile__()
157
158	return
159
160	def __read_controlfile__(self):
161	'''Read CONTROL file and assign all CONTROL file variables.
162
163	Parameters
164	----------
165
166	Return
167	------
168
169	'''
170
171	try:
172	cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
173	with open(cfile) as f:
174	fdata = f.read().split('\n')
175	except IOError:
176	print('Could not read CONTROL file "' + cfile + '"')
177	print('Either it does not exist or its syntax is wrong.')
178	print('Try "' + sys.argv[0].split('/')[-1] + \
179	' -h" to print usage information')
180	sys.exit(1)
181
182	# go through every line and store parameter
183	for ldata in fdata:
184	data = ldata.split()
185	if len(data) > 1:
186	if 'm_' in data[0].lower():
187	data[0] = data[0][2:]
188	if data[0].lower() == 'class':
189	data[0] = 'marsclass'
190	if data[0].lower() == 'day1':
191	data[0] = 'start_date'
192	if data[0].lower() == 'day2':
193	data[0] = 'end_date'
194	if data[0].lower() == 'addpar':
195	if '/' in data[1]:
196	# remove leading '/' sign from addpar content
197	if data[1][0] == '/':
198	data[1] = data[1][1:]
199	dd = data[1].split('/')
200	data = [data[0]]
201	for d in dd:
202	data.append(d)
203	if len(data) == 2:
204	if '$' in data[1]:
205	setattr(self, data[0].lower(), data[1])
206	while '$' in data[1]:
207	i = data[1].index('$')
208	j = data[1].find('{')
209	k = data[1].find('}')
210	var = os.getenv(data[1][j+1:k])
211	if var is not None:
212	data[1] = data[1][:i] + var + data[1][k+1:]
213	else:
214	my_error(self.mailfail,
215	'Could not find variable '
216	+ data[1][j+1:k] + ' while reading ' +
217	self.controlfile)
218	setattr(self, data[0].lower() + '_expanded', data[1])
219	else:
220	if data[1].lower() != 'none':
221	setattr(self, data[0].lower(), data[1])
222	else:
223	setattr(self, data[0].lower(), None)
224	elif len(data) > 2:
225	setattr(self, data[0].lower(), (data[1:]))
226	else:
227	pass
228
229	return
230
231	def __str__(self):
232	'''Prepares a string which have all the ControlFile class attributes
233	with its associated values. Each attribute is printed in one line and
234	in alphabetical order.
235
236	Example
237	-------
238	'age': 10
239	'color': 'Spotted'
240	'kids': 0
241	'legs': 2
242	'name': 'Dog'
243	'smell': 'Alot'
244
245	Parameters
246	----------
247
248	Return
249	------
250	string
251	Single string of concatenated ControlFile class attributes
252	with their values
253	'''
254	import collections
255
256	attrs = vars(self).copy()
257	attrs = collections.OrderedDict(sorted(attrs.items()))
258
259	return '\n'.join("%s: %s" % item for item in attrs.items())
260
261	def assign_args_to_control(self, args):
262	'''Overwrites the existing ControlFile instance attributes with
263	the command line arguments.
264
265	Parameters
266	----------
267	args : :obj:`Namespace`
268	Contains the commandline arguments from script/program call.
269
270	Return
271	------
272
273	'''
274
275	# get dictionary of command line parameters and eliminate all
276	# parameters which are None (were not specified)
277	args_dict = vars(args)
278	arguments = {k : args_dict[k] for k in args_dict
279	if args_dict[k] != None}
280
281	# assign all passed command line arguments to ControlFile instance
282	for k, v in arguments.iteritems():
283	setattr(self, str(k), v)
284
285	return
286
287	def assign_envs_to_control(self, envs):
288	'''Assigns the ECMWF environment parameter.
289
290	Parameters
291	----------
292	envs : :obj:`dictionary` of :obj:`strings`
293	Contains the ECMWF environment parameternames "ECUID", "ECGID",
294	"DESTINATION" and "GATEWAY" with its corresponding values.
295	They were read from the file "ECMWF_ENV".
296
297	Return
298	------
299
300	'''
301
302	for k, v in envs.iteritems():
303	setattr(self, str(k).lower(), str(v))
304
305	return
306
307	def check_conditions(self, queue):
308	'''Checks a couple of necessary attributes and conditions,
309	such as if they exist and contain values.
310	Otherwise set default values.
311
312	Parameters
313	----------
314	queue : :obj:`string`
315	Name of the queue if submitted to the ECMWF servers.
316	Used to check if ecuid, ecgid, gateway and destination
317	are set correctly and are not empty.
318
319	Return
320	------
321
322	'''
323	from mods.tools import my_error
324	import numpy as np
325
326	# check for having at least a starting date
327	# otherwise program is not allowed to run
328	if not self.start_date:
329	print('start_date specified neither in command line nor '
330	'in CONTROL file ' + self.controlfile)
331	print('Try "' + sys.argv[0].split('/')[-1] +
332	' -h" to print usage information')
333	sys.exit(1)
334
335	# retrieve just one day if end_date isn't set
336	if not self.end_date:
337	self.end_date = self.start_date
338
339	# basetime has only two possible values
340	if self.basetime:
341	if int(self.basetime) != 0 and int(self.basetime) != 12:
342	print('Basetime has an invalid value!')
343	print('Basetime = ' + str(self.basetime))
344	sys.exit(1)
345
346	# assure consistency of levelist and level
347	if not self.levelist and not self.level:
348	print('Warning: neither levelist nor level \
349	specified in CONTROL file')
350	sys.exit(1)
351	elif not self.levelist and self.level:
352	self.levelist = '1/to/' + self.level
353	elif (self.levelist and not self.level) or \
354	(self.levelist[-1] != self.level[-1]):
355	self.level = self.levelist.split('/')[-1]
356	else:
357	pass
358
359	# check if max level is a valid level
360	if int(self.level) not in _config.MAX_LEVEL_LIST:
361	print('ERROR: ')
362	print('LEVEL must be the maximum level of a specified '
363	'level list from ECMWF, e.g.')
364	print(_config.MAX_LEVEL_LIST)
365	print('Check parameter "LEVEL" or the max level of "LEVELIST"!')
366	sys.exit(1)
367
368	# prepare step list if "/" signs are found
369	if '/' in self.step:
370	steps = self.step.split('/')
371	if 'to' in self.step.lower() and 'by' in self.step.lower():
372	ilist = np.arange(int(steps[0]),
373	int(steps[2]) + 1,
374	int(steps[4]))
375	self.step = ['{:0>3}'.format(i) for i in ilist]
376	elif 'to' in self.step.lower() and 'by' not in self.step.lower():
377	my_error(self.mailfail, self.step + ':\n' +
378	'if "to" is used in steps parameter, '
379	'please use "by" as well')
380	else:
381	self.step = steps
382
383	# if maxstep wasn't provided
384	# search for it in the "step" parameter
385	if not self.maxstep:
386	self.maxstep = 0
387	for s in self.step:
388	if int(s) > self.maxstep:
389	self.maxstep = int(s)
390	else:
391	self.maxstep = int(self.maxstep)
392
393	# set root scripts since it is needed later on
394	if not self.flexpart_root_scripts:
395	self.flexpart_root_scripts = self.ecmwfdatadir
396
397	if not self.outputdir:
398	self.outputdir = self.inputdir
399
400	if not isinstance(self.mailfail, list):
401	if ',' in self.mailfail:
402	self.mailfail = self.mailfail.split(',')
403	elif ' ' in self.mailfail:
404	self.mailfail = self.mailfail.split()
405	else:
406	self.mailfail = [self.mailfail]
407
408	if not isinstance(self.mailops, list):
409	if ',' in self.mailops:
410	self.mailops = self.mailops.split(',')
411	elif ' ' in self.mailops:
412	self.mailops = self.mailops.split()
413	else:
414	self.mailops = [self.mailops]
415
416	if queue in _config.QUEUES_LIST and \
417	not self.gateway or not self.destination or \
418	not self.ecuid or not self.ecgid:
419	print('\nEnvironment variables GATEWAY, DESTINATION, ECUID and '
420	'ECGID were not set properly!')
421	print('Please check for existence of file "ECMWF_ENV" in the '
422	'python directory!')
423	sys.exit(1)
424
425	if self.request != 0:
426	marsfile = os.path.join(self.inputdir,
427	_config.FILE_MARS_REQUESTS)
428	if os.path.isfile(marsfile):
429	silent_remove(marsfile)
430
431	# check all logical variables for data type
432	# if its a string change to integer
433	for var in self.logicals:
434	if not isinstance(getattr(self, var), int):
435	setattr(self, var, int(getattr(self, var)))
436
437	if self.public and not self.dataset:
438	print('ERROR: ')
439	print('If public mars data wants to be retrieved, '
440	'the "dataset"-parameter has to be set in the control file!')
441	sys.exit(1)
442
443	if not isinstance(self.type, list):
444	self.type = [self.type]
445
446	for i, val in enumerate(self.type):
447	if self.type[i] == 'AN' and int(self.step[i]) != 0:
448	print('Analysis retrievals must have STEP = 0 (is set to 0)')
449	self.type[i] = 0
450
451	if not isinstance(self.time, list):
452	self.time = [self.time]
453
454	if not isinstance(self.step, list):
455	self.step = [self.step]
456
457	if not self.acctype:
458	print('... Control paramter ACCTYPE was not defined.')
459	try:
460	if len(self.type) > 1 and self.type[1] != 'AN':
461	print('Use old setting by using TYPE[1] for flux forecast!')
462	self.acctype = self.type[1]
463	except:
464	print('Use default value "FC" for flux forecast!')
465	self.acctype='FC'
466
467	if not self.acctime:
468	print('... Control paramter ACCTIME was not defined.')
469	print('Use default value "00/12" for flux forecast!')
470	self.acctime='00/12'
471
472	if not self.accmaxstep:
473	print('... Control paramter ACCMAXSTEP was not defined.')
474	print('Use default value "12" for flux forecast!')
475	self.accmaxstep='12'
476
477	# if area was provided (only from commandline)
478	# decompose area into its 4 components
479	if self.area:
480	components = self.area.split('/')
481	self.upper, self.left, self.lower, self.right = components
482
483	# convert grid and area components to correct format and input
484	if 'N' in self.grid: # Gaussian output grid
485	self.area = 'G'
486	else:
487	# check on grid format
488	if float(self.grid) / 100. >= 0.5:
489	# grid is defined in 1/1000 degrees; old method
490	self.grid = '{}/{}'.format(float(self.grid) / 1000.,
491	float(self.grid) / 1000.)
492	self.area = '{}/{}/{}/{}'.format(float(self.upper) / 1000.,
493	float(self.left) / 1000.,
494	float(self.lower) / 1000.,
495	float(self.right) / 1000.)
496	elif float(self.grid) / 100. < 0.5:
497	# grid is defined in normal degree; new method
498	self.grid = '{}/{}'.format(float(self.grid), float(self.grid))
499	self.area = '{}/{}/{}/{}'.format(float(self.upper),
500	float(self.left),
501	float(self.lower),
502	float(self.right))
503
504	return
505
506	def check_install_conditions(self):
507	'''Checks a couple of necessary attributes and conditions
508	for the installation such as if they exist and contain values.
509	Otherwise set default values.
510
511	Parameters
512	----------
513
514	Return
515	------
516
517	'''
518
519	if self.install_target and \
520	self.install_target not in ['local', 'ecgate', 'cca']:
521	print('ERROR: unknown or missing installation target ')
522	print('target: ', self.install_target)
523	print('please specify correct installation target ' +
524	'(local \| ecgate \| cca)')
525	print('use -h or --help for help')
526	sys.exit(1)
527
528	if self.install_target and self.install_target != 'local':
529	if not self.ecgid or not self.ecuid or \
530	not self.gateway or not self.destination:
531	print('Please enter your ECMWF user id and group id as well ' +
532	'as the \nname of the local gateway and the ectrans ' +
533	'destination ')
534	print('with command line options --ecuid --ecgid \
535	--gateway --destination')
536	print('Try "' + sys.argv[0].split('/')[-1] + \
537	' -h" to print usage information')
538	print('Please consult ecaccess documentation or ECMWF user \
539	support for further details')
540	sys.exit(1)
541
542	if not self.flexpart_root_scripts:
543	self.flexpart_root_scripts = '${HOME}'
544	else:
545	self.flexpart_root_scripts = self.flexpart_root_scripts
546	else: # local
547	if not self.flexpart_root_scripts:
548	self.flexpart_root_scripts = _config.PATH_FLEXEXTRACT_DIR
549
550	return
551
552	def to_list(self):
553	'''Just generates a list of strings containing the attributes and
554	assigned values except the attributes "_expanded", "exedir",
555	"ecmwfdatadir" and "flexpart_root_scripts".
556
557	Parameters
558	----------
559
560	Return
561	------
562	l : :obj:`list`
563	A sorted list of the all ControlFile class attributes with
564	their values except the attributes "_expanded", "exedir",
565	"ecmwfdatadir" and "flexpart_root_scripts".
566	'''
567
568	import collections
569
570	attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
571
572	l = list()
573
574	for item in attrs.items():
575	if '_expanded' in item[0]:
576	pass
577	elif 'exedir' in item[0]:
578	pass
579	elif 'flexpart_root_scripts' in item[0]:
580	pass
581	elif 'ecmwfdatadir' in item[0]:
582	pass
583	else:
584	if isinstance(item[1], list):
585	stot = ''
586	for s in item[1]:
587	stot += s + ' '
588
589	l.append("%s %s" % (item[0], stot))
590	else:
591	l.append("%s %s" % item)
592
593	return sorted(l)
594

Note: See TracBrowser for help on using the repository browser.

Download in other formats: