Context Navigation

source: flex_extract.git/source/python/classes/ControlFile.py @ 97f4f4c

ctbtodev

Last change on this file since 97f4f4c was 97f4f4c, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
better check on grid and arae parameter formats
Property mode set to `100644`
File size: 19.5 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - applied some minor modifications in programming style/structure
14	# - changed name of class Control to ControlFile for more
15	# self-explanation naming
16	# - outsource of class ControlFile
17	# - initialisation of class attributes ( to avoid high number of
18	# conditional statements and set default values )
19	# - divided assignment of attributes and the check of conditions
20	# - outsourced the commandline argument assignments to control attributes
21	#
22	# @License:
23	# (C) Copyright 2015-2018.
24	#
25	# This software is licensed under the terms of the Apache Licence Version 2.0
26	# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
27	#
28	# @Class Description:
29	# The CONTROL file is the steering part of the FLEXPART extraction
30	# software. All necessary parameters needed to retrieve the data fields
31	# from the MARS archive for driving FLEXPART are set in a CONTROL file.
32	# Some specific parameters like the start and end dates can be overwritten
33	# by the command line parameters, but in generel all parameters needed
34	# for a complete set of fields for FLEXPART can be set in the CONTROL file.
35	#
36	# @Class Content:
37	# - __init__
38	# - __read_controlfile__
39	# - __str__
40	# - assign_args_to_control
41	# - assign_envs_to_control
42	# - check_conditions
43	# - check_install_conditions
44	# - to_list
45	#
46	# @Class Attributes:
47	#
48	#
49	#*******************************************************************************
50
51	# ------------------------------------------------------------------------------
52	# MODULES
53	# ------------------------------------------------------------------------------
54	import os
55	import re
56	import sys
57	import inspect
58
59	# software specific classes and modules from flex_extract
60	sys.path.append('../')
61	import _config
62	from mods.tools import my_error, silent_remove
63	from mods.checks import check_grid_area
64
65	# ------------------------------------------------------------------------------
66	# CLASS
67	# ------------------------------------------------------------------------------
68	class ControlFile(object):
69	'''
70	Contains the information which are stored in the CONTROL files.
71	'''
72
73	def __init__(self, filename):
74	'''Initialises the instance of ControlFile class and defines
75	all class attributes with default values. Afterwards calls
76	function __read_controlfile__ to read parameter from Control file.
77
78	Parameters
79	----------
80	filename : :obj:`string`
81	Name of CONTROL file.
82
83	Return
84	------
85
86	'''
87
88	# list of all possible class attributes and their default values
89	self.controlfile = filename
90	self.start_date = None
91	self.end_date = None
92	self.date_chunk = 3
93	self.dtime = None
94	self.basetime = None
95	self.maxstep = None
96	self.type = None
97	self.time = None
98	self.step = None
99	self.acctype = None
100	self.acctime = None
101	self.accmaxstep = None
102	self.marsclass = None
103	self.dataset = None
104	self.stream = None
105	self.number = 'OFF'
106	self.expver = '1'
107	self.gaussian = ''
108	self.grid = None
109	self.area = ''
110	self.left = None
111	self.lower = None
112	self.upper = None
113	self.right = None
114	self.level = None
115	self.levelist = None
116	self.resol = None
117	self.gauss = 0
118	self.accuracy = 24
119	self.omega = 0
120	self.omegadiff = 0
121	self.eta = 0
122	self.etadiff = 0
123	self.etapar = 77
124	self.dpdeta = 1
125	self.smooth = 0
126	self.format = 'GRIB1'
127	self.addpar = None
128	self.prefix = 'EN'
129	self.cwc = 0
130	self.wrf = 0
131	self.ecfsdir = 'ectmp:/${USER}/econdemand/'
132	self.mailfail = ['${USER}']
133	self.mailops = ['${USER}']
134	self.grib2flexpart = 0
135	self.ecstorage = 0
136	self.ectrans = 0
137	self.inputdir = _config.PATH_INPUT_DIR
138	self.outputdir = None
139	self.ecmwfdatadir = _config.PATH_FLEXEXTRACT_DIR
140	self.exedir = _config.PATH_FORTRAN_SRC
141	self.flexpart_root_scripts = None
142	self.makefile = 'Makefile.gfortran'
143	self.destination = None
144	self.gateway = None
145	self.ecuid = None
146	self.ecgid = None
147	self.install_target = None
148	self.debug = 0
149	self.request = 0
150	self.public = 0
151	self.ecapi = None
152
153	self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
154	'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
155	'ectrans', 'debug', 'request', 'public']
156
157	self.__read_controlfile__()
158
159	return
160
161	def __read_controlfile__(self):
162	'''Read CONTROL file and assign all CONTROL file variables.
163
164	Parameters
165	----------
166
167	Return
168	------
169
170	'''
171
172	try:
173	cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
174	with open(cfile) as f:
175	fdata = f.read().split('\n')
176	except IOError:
177	print('Could not read CONTROL file "' + cfile + '"')
178	print('Either it does not exist or its syntax is wrong.')
179	print('Try "' + sys.argv[0].split('/')[-1] + \
180	' -h" to print usage information')
181	sys.exit(1)
182
183	# go through every line and store parameter
184	for ldata in fdata:
185	data = ldata.split()
186	if len(data) > 1:
187	if 'm_' in data[0].lower():
188	data[0] = data[0][2:]
189	if data[0].lower() == 'class':
190	data[0] = 'marsclass'
191	if data[0].lower() == 'day1':
192	data[0] = 'start_date'
193	if data[0].lower() == 'day2':
194	data[0] = 'end_date'
195	if data[0].lower() == 'addpar':
196	if '/' in data[1]:
197	# remove leading '/' sign from addpar content
198	if data[1][0] == '/':
199	data[1] = data[1][1:]
200	dd = data[1].split('/')
201	data = [data[0]]
202	for d in dd:
203	data.append(d)
204	if len(data) == 2:
205	if '$' in data[1]:
206	setattr(self, data[0].lower(), data[1])
207	while '$' in data[1]:
208	i = data[1].index('$')
209	j = data[1].find('{')
210	k = data[1].find('}')
211	var = os.getenv(data[1][j+1:k])
212	if var is not None:
213	data[1] = data[1][:i] + var + data[1][k+1:]
214	else:
215	my_error(self.mailfail,
216	'Could not find variable '
217	+ data[1][j+1:k] + ' while reading ' +
218	self.controlfile)
219	setattr(self, data[0].lower() + '_expanded', data[1])
220	else:
221	if data[1].lower() != 'none':
222	setattr(self, data[0].lower(), data[1])
223	else:
224	setattr(self, data[0].lower(), None)
225	elif len(data) > 2:
226	setattr(self, data[0].lower(), (data[1:]))
227	else:
228	pass
229
230	return
231
232	def __str__(self):
233	'''Prepares a string which have all the ControlFile class attributes
234	with its associated values. Each attribute is printed in one line and
235	in alphabetical order.
236
237	Example
238	-------
239	'age': 10
240	'color': 'Spotted'
241	'kids': 0
242	'legs': 2
243	'name': 'Dog'
244	'smell': 'Alot'
245
246	Parameters
247	----------
248
249	Return
250	------
251	string
252	Single string of concatenated ControlFile class attributes
253	with their values
254	'''
255	import collections
256
257	attrs = vars(self).copy()
258	attrs = collections.OrderedDict(sorted(attrs.items()))
259
260	return '\n'.join("%s: %s" % item for item in attrs.items())
261
262	def assign_args_to_control(self, args):
263	'''Overwrites the existing ControlFile instance attributes with
264	the command line arguments.
265
266	Parameters
267	----------
268	args : :obj:`Namespace`
269	Contains the commandline arguments from script/program call.
270
271	Return
272	------
273
274	'''
275
276	# get dictionary of command line parameters and eliminate all
277	# parameters which are None (were not specified)
278	args_dict = vars(args)
279	arguments = {k : args_dict[k] for k in args_dict
280	if args_dict[k] != None}
281
282	# assign all passed command line arguments to ControlFile instance
283	for k, v in arguments.iteritems():
284	setattr(self, str(k), v)
285
286	return
287
288	def assign_envs_to_control(self, envs):
289	'''Assigns the ECMWF environment parameter.
290
291	Parameters
292	----------
293	envs : :obj:`dictionary` of :obj:`strings`
294	Contains the ECMWF environment parameternames "ECUID", "ECGID",
295	"DESTINATION" and "GATEWAY" with its corresponding values.
296	They were read from the file "ECMWF_ENV".
297
298	Return
299	------
300
301	'''
302
303	for k, v in envs.iteritems():
304	setattr(self, str(k).lower(), str(v))
305
306	return
307
308	def check_conditions(self, queue):
309	'''Checks a couple of necessary attributes and conditions,
310	such as if they exist and contain values.
311	Otherwise set default values.
312
313	Parameters
314	----------
315	queue : :obj:`string`
316	Name of the queue if submitted to the ECMWF servers.
317	Used to check if ecuid, ecgid, gateway and destination
318	are set correctly and are not empty.
319
320	Return
321	------
322
323	'''
324	from mods.tools import my_error
325	import numpy as np
326
327	# check for having at least a starting date
328	# otherwise program is not allowed to run
329	if not self.start_date:
330	print('start_date specified neither in command line nor '
331	'in CONTROL file ' + self.controlfile)
332	print('Try "' + sys.argv[0].split('/')[-1] +
333	' -h" to print usage information')
334	sys.exit(1)
335
336	# retrieve just one day if end_date isn't set
337	if not self.end_date:
338	self.end_date = self.start_date
339
340	# basetime has only two possible values
341	if self.basetime:
342	if int(self.basetime) != 0 and int(self.basetime) != 12:
343	print('Basetime has an invalid value!')
344	print('Basetime = ' + str(self.basetime))
345	sys.exit(1)
346
347	# assure consistency of levelist and level
348	if not self.levelist and not self.level:
349	print('Warning: neither levelist nor level \
350	specified in CONTROL file')
351	sys.exit(1)
352	elif not self.levelist and self.level:
353	self.levelist = '1/to/' + self.level
354	elif (self.levelist and not self.level) or \
355	(self.levelist[-1] != self.level[-1]):
356	self.level = self.levelist.split('/')[-1]
357	else:
358	pass
359
360	# check if max level is a valid level
361	if int(self.level) not in _config.MAX_LEVEL_LIST:
362	print('ERROR: ')
363	print('LEVEL must be the maximum level of a specified '
364	'level list from ECMWF, e.g.')
365	print(_config.MAX_LEVEL_LIST)
366	print('Check parameter "LEVEL" or the max level of "LEVELIST"!')
367	sys.exit(1)
368
369	# prepare step list if "/" signs are found
370	if '/' in self.step:
371	steps = self.step.split('/')
372	if 'to' in self.step.lower() and 'by' in self.step.lower():
373	ilist = np.arange(int(steps[0]),
374	int(steps[2]) + 1,
375	int(steps[4]))
376	self.step = ['{:0>3}'.format(i) for i in ilist]
377	elif 'to' in self.step.lower() and 'by' not in self.step.lower():
378	my_error(self.mailfail, self.step + ':\n' +
379	'if "to" is used in steps parameter, '
380	'please use "by" as well')
381	else:
382	self.step = steps
383
384	# if maxstep wasn't provided
385	# search for it in the "step" parameter
386	if not self.maxstep:
387	self.maxstep = 0
388	for s in self.step:
389	if int(s) > self.maxstep:
390	self.maxstep = int(s)
391	else:
392	self.maxstep = int(self.maxstep)
393
394	# set root scripts since it is needed later on
395	if not self.flexpart_root_scripts:
396	self.flexpart_root_scripts = self.ecmwfdatadir
397
398	if not self.outputdir:
399	self.outputdir = self.inputdir
400
401	if not isinstance(self.mailfail, list):
402	if ',' in self.mailfail:
403	self.mailfail = self.mailfail.split(',')
404	elif ' ' in self.mailfail:
405	self.mailfail = self.mailfail.split()
406	else:
407	self.mailfail = [self.mailfail]
408
409	if not isinstance(self.mailops, list):
410	if ',' in self.mailops:
411	self.mailops = self.mailops.split(',')
412	elif ' ' in self.mailops:
413	self.mailops = self.mailops.split()
414	else:
415	self.mailops = [self.mailops]
416
417	if queue in _config.QUEUES_LIST and \
418	not self.gateway or not self.destination or \
419	not self.ecuid or not self.ecgid:
420	print('\nEnvironment variables GATEWAY, DESTINATION, ECUID and '
421	'ECGID were not set properly!')
422	print('Please check for existence of file "ECMWF_ENV" in the '
423	'python directory!')
424	sys.exit(1)
425
426	if self.request != 0:
427	marsfile = os.path.join(self.inputdir,
428	_config.FILE_MARS_REQUESTS)
429	if os.path.isfile(marsfile):
430	silent_remove(marsfile)
431
432	# check all logical variables for data type
433	# if its a string change to integer
434	for var in self.logicals:
435	if not isinstance(getattr(self, var), int):
436	setattr(self, var, int(getattr(self, var)))
437
438	if self.public and not self.dataset:
439	print('ERROR: ')
440	print('If public mars data wants to be retrieved, '
441	'the "dataset"-parameter has to be set in the control file!')
442	sys.exit(1)
443
444	if not isinstance(self.type, list):
445	self.type = [self.type]
446
447	for i, val in enumerate(self.type):
448	if self.type[i] == 'AN' and int(self.step[i]) != 0:
449	print('Analysis retrievals must have STEP = 0 (is set to 0)')
450	self.type[i] = 0
451
452	if not isinstance(self.time, list):
453	self.time = [self.time]
454
455	if not isinstance(self.step, list):
456	self.step = [self.step]
457
458	if not self.acctype:
459	print('... Control paramter ACCTYPE was not defined.')
460	try:
461	if len(self.type) > 1 and self.type[1] != 'AN':
462	print('Use old setting by using TYPE[1] for flux forecast!')
463	self.acctype = self.type[1]
464	except:
465	print('Use default value "FC" for flux forecast!')
466	self.acctype='FC'
467
468	if not self.acctime:
469	print('... Control paramter ACCTIME was not defined.')
470	print('Use default value "00/12" for flux forecast!')
471	self.acctime='00/12'
472
473	if not self.accmaxstep:
474	print('... Control paramter ACCMAXSTEP was not defined.')
475	print('Use default value "12" for flux forecast!')
476	self.accmaxstep='12'
477
478
479	self.grid, self.area = check_grid_area(self.grid, self.area,
480	self.upper, self.lower,
481	self.left, self.right)
482
483
484
485	# convert grid and area components to correct format and input
486	#if 'N' in self.grid: # Gaussian output grid
487	# self.area = 'G'
488	# else:
489	# if '/' in self.grid:
490	# gridx, gridy = self.grid.split('/')
491	# if gridx == gridy:
492	# self.grid = gridx
493
494
495	# # check on grid format
496	# if float(self.grid) / 100. >= 0.5:
497	# # grid is defined in 1/1000 degrees; old format
498	# self.grid = '{}/{}'.format(float(self.grid) / 1000.,
499	# float(self.grid) / 1000.)
500	# self.area = '{}/{}/{}/{}'.format(float(self.upper) / 1000.,
501	# float(self.left) / 1000.,
502	# float(self.lower) / 1000.,
503	# float(self.right) / 1000.)
504	# elif float(self.grid) / 100. < 0.5:
505	# # grid is defined in normal degree; new format
506	# self.grid = '{}/{}'.format(float(self.grid), float(self.grid))
507	# self.area = '{}/{}/{}/{}'.format(float(self.upper),
508	# float(self.left),
509	# float(self.lower),
510	# float(self.right))
511
512	return
513
514	def to_list(self):
515	'''Just generates a list of strings containing the attributes and
516	assigned values except the attributes "_expanded", "exedir",
517	"ecmwfdatadir" and "flexpart_root_scripts".
518
519	Parameters
520	----------
521
522	Return
523	------
524	l : :obj:`list`
525	A sorted list of the all ControlFile class attributes with
526	their values except the attributes "_expanded", "exedir",
527	"ecmwfdatadir" and "flexpart_root_scripts".
528	'''
529
530	import collections
531
532	attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
533
534	l = list()
535
536	for item in attrs.items():
537	if '_expanded' in item[0]:
538	pass
539	elif 'exedir' in item[0]:
540	pass
541	elif 'flexpart_root_scripts' in item[0]:
542	pass
543	elif 'ecmwfdatadir' in item[0]:
544	pass
545	else:
546	if isinstance(item[1], list):
547	stot = ''
548	for s in item[1]:
549	stot += s + ' '
550
551	l.append("%s %s\n" % (item[0], stot))
552	else:
553	l.append("%s %s\n" % item)
554
555	return sorted(l)
556
557	def check_ppid(self, ppid):
558	'''Sets the current PPID.
559
560	Parameters
561	----------
562	ppid : :obj:`int` or :obj:`None`
563	Contains the ppid number provided by the command line parameter
564	of is None otherwise.
565
566	Return
567	------
568
569	'''
570
571	if not ppid:
572	self.ppid = str(os.getppid())
573	else:
574	self.ppid = ppid
575
576	return

Note: See TracBrowser for help on using the repository browser.

Download in other formats: