Context Navigation

source: flex_extract.git/Source/Python/Classes/ControlFile.py @ 0f89116

ctbtodev

Last change on this file since 0f89116 was 0f89116, checked in by Anne Philipp <anne.philipp@…>, 4 years ago
diverse changes due to PEP8 style guide and eliminating grib2flexpart; removed unused parameter
Property mode set to `100644`
File size: 22.7 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - applied some minor modifications in programming style/structure
14	# - changed name of class Control to ControlFile for more
15	# self-explanation naming
16	# - outsource of class ControlFile
17	# - initialisation of class attributes ( to avoid high number of
18	# conditional statements and set default values )
19	# - divided assignment of attributes and the check of conditions
20	# - outsourced the commandline argument assignments to control attributes
21	#
22	# @License:
23	# (C) Copyright 2014-2019.
24	# Anne Philipp, Leopold Haimberger
25	#
26	# SPDX-License-Identifier: CC-BY-4.0
27	#
28	# This work is licensed under the Creative Commons Attribution 4.0
29	# International License. To view a copy of this license, visit
30	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
31	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
32	#*******************************************************************************
33
34	# ------------------------------------------------------------------------------
35	# MODULES
36	# ------------------------------------------------------------------------------
37	from __future__ import print_function
38
39	import os
40	import sys
41
42	# software specific classes and modules from flex_extract
43	#pylint: disable=wrong-import-position
44	sys.path.append('../')
45	import _config
46	from Mods.tools import my_error
47	from Mods.checks import (check_grid, check_area, check_levels, check_purefc,
48	check_step, check_mail, check_queue, check_pathes,
49	check_dates, check_maxstep, check_type, check_request,
50	check_basetime, check_public, check_acctype,
51	check_acctime, check_accmaxstep, check_time,
52	check_logicals_type, check_len_type_time_step,
53	check_addpar, check_job_chunk, check_number)
54	#pylint: enable=wrong-import-position
55
56	# ------------------------------------------------------------------------------
57	# CLASS
58	# ------------------------------------------------------------------------------
59	class ControlFile(object):
60	'''
61	Contains the information which are stored in the CONTROL files.
62
63	The CONTROL file is the steering part of the FLEXPART extraction
64	software. All necessary parameters needed to retrieve the data fields
65	from the MARS archive for driving FLEXPART are set in a CONTROL file.
66	Some specific parameters like the start and end dates can be overwritten
67	by the command line parameters, but in generall all parameters needed
68	for a complete set of fields for FLEXPART can be set in the CONTROL file.
69
70	Attributes
71	----------
72	controlfile : str
73	The name of the control file to be processed. Default value is the
74	filename passed to the init function when initialised.
75
76	start_date : str
77	The first day of the retrieval period. Default value is None.
78
79	end_date :str
80	The last day of the retrieval period. Default value is None.
81
82	date_chunk : int
83	Length of period for a single mars retrieval. Default value is 3.
84
85	dtime :str
86	The time step in hours. Default value is None.
87
88	basetime : int
89	The time for a half day retrieval. The 12 hours upfront are to be
90	retrieved. Default value is None.
91
92	maxstep : int
93	The maximum forecast step for non flux data. Default value is None.
94
95	type : list of str
96	List of field type per retrieving hour. Default value is None.
97
98	time : list of str
99	List of retrieving times in hours. Default valuer is None.
100
101	step : list of str or str
102	List of forecast time steps in hours for non flux data.
103	Default value is None.
104
105	acctype : str
106	The field type for the accumulated forecast fields.
107	Default value is None.
108
109	acctime : str
110	The starting time of the accumulated forecasts. Default value is None.
111
112	accmaxstep : int
113	The maximum forecast step for the accumulated forecast fields
114	(flux data). Default value is None.
115
116	marsclass : str
117	Characterisation of dataset. Default value is None.
118
119	dataset : str
120	For public datasets there is the specific naming and parameter
121	dataset which has to be used to characterize the type of
122	data. Default value is None.
123
124	stream : str
125	Identifies the forecasting system used to generate the data.
126	Default value is None.
127
128	number : str
129	Selects the member in ensemble forecast run. Default value is 'OFF'.
130
131	expver : str
132	The version number of the dataset. Default value is '1'.
133
134	gaussian : str
135	This parameter is deprecated and should no longer be used.
136	Specifies the desired type of Gaussian grid for the output.
137	Default value is an empty string ''.
138
139	grid : str
140	Specifies the output grid which can be either a Gaussian grid
141	or a Latitude/Longitude grid. Default value is None.
142
143	area : str
144	Specifies the desired sub-area of data to be extracted.
145	Default value is None.
146
147	left : str
148	The western most longitude of the area to be extracted.
149	Default value is None.
150
151	lower : str
152	The southern most latitude of the area to be extracted.
153	Default value is None.
154
155	upper : str
156	The northern most latitued of the area to be extracted.
157	Default value is None.
158
159	right : str
160	The eastern most longitude of the area to be extracted.
161	Default value is None.
162
163	level : str
164	Specifies the maximum level. Default value is None.
165
166	levelist : str
167	Specifies the required level list. Default value is None.
168
169	resol : str
170	Specifies the desired triangular truncation of retrieved data,
171	before carrying out any other selected post-processing.
172	Default value is None.
173
174	gauss : int
175	Switch to select gaussian fields (1) or regular lat/lon (0).
176	Default value is 0.
177
178	accuracy : int
179	Specifies the number of bits per value to be used in the
180	generated GRIB coded fields. Default value is 24.
181
182	omega : int
183	Switch to select omega retrieval (1) or not (0). Default value is 0.
184
185	omegadiff : int
186	Switch to decide to calculate Omega and Dps/Dt from continuity
187	equation for diagnostic purposes (1) or not (0). Default value is 0.
188
189	eta : int
190	Switch to select direct retrieval of etadot from MARS (1) or
191	wether it has to be calculated (0). Then Default value is 0.
192
193	etadiff : int
194	Switch to select calculation of etadot and Dps/Dt from continuity
195	equation for diagnostic purposes (1) or not (0). Default value is 0.
196
197	etapar : int
198	GRIB parameter Id for etadot fields. Default value is 77.
199
200	dpdeta : int
201	Switch to select multiplication of etadot with dpdeta.
202	Default value is 1.
203
204	smooth : int
205	Spectral truncation of ETADOT after calculation on Gaussian grid.
206	Default value is 0.
207
208	format : str
209	The format of the GRIB data. Default value is 'GRIB1'.
210
211	addpar : str
212	List of additional surface level ECMWF parameter to be retrieved.
213	Default value is None.
214
215	prefix : str
216	Prefix string for the final FLEXPART/FLEXTRA ready input files.
217	Default value is 'EN'.
218
219	cwc : int
220	Switch to select wether the sum of cloud liquid water content and
221	cloud ice water content should be retrieved. Default value is 0.
222
223	wrf : int
224	Switch to select further parameters for retrievment to support
225	WRF simulations. Default value is 0.
226
227	ecfsdir : str
228	Path to the ECMWF storage 'ectmp:/${USER}/econdemand/'
229
230	mailfail : list of str
231	Email list for sending error log files from ECMWF servers.
232	The email addresses should be seperated by a comma.
233	Default value is ['${USER}'].
234
235	mailops : list of str
236	Email list for sending operational log files from ECMWF servers.
237	The email addresses should be seperated by a comma.
238	Default value is ['${USER}'].
239
240	ecstorage : int
241	Switch to select storage of FLEXPART ready output files
242	in the ECFS file system. Default value is 0.
243
244	ectrans : int
245	Switch to select the transfer of FLEXPART ready output files
246	to the gateway server. Default value is 0.
247
248	inputdir : str
249	Path to the temporary directory for the retrieval grib files and
250	other processing files. Default value is _config.PATH_INPUT_DIR.
251
252	outputdir : str
253	Path to the final directory where the final FLEXPART ready input
254	files are stored. Default value is None.
255
256	flexextractdir : str
257	Path to the flex_extract root directory. Default value is
258	_config.PATH_FLEXEXTRACT_DIR.
259
260	exedir : str
261	Path to the FORTRAN executable file. Default value is
262	_config.PATH_FORTRAN_SRC.
263
264	installdir : str
265	Path to a FLEXPART root directory. Default value is None.
266
267	makefile : str
268	Name of the makefile to be used for the Fortran program.
269	Default value is 'Makefile.gfortran'.
270
271	destination : str
272	The remote destination which is used to transfer files
273	from ECMWF server to local gateway server. Default value is None.
274
275	gateway : str
276	The gateway server the user is using. Default value is None.
277
278	ecuid : str
279	The user id on ECMWF server. Default value is None.
280
281	ecgid : str
282	The group id on ECMWF server. Default value is None.
283
284	install_target : str
285	Defines the location where the installation is to be done.
286	Default value is None.
287
288	debug : int
289	Switch to keep temporary files at the end of postprocessing (1) or
290	to delete all temporary files except the final output files (0).
291	Default value is 0.
292
293	oper : int
294	Switch to prepare the operational job script. Start date, end date and
295	basetime will be prepared with environment variables.
296	Default value is 0.
297
298	request : int
299	Switch to select between just retrieving the data (0), writing the mars
300	parameter values to a csv file (1) or doing both (2).
301	Default value is 0.
302
303	public : int
304	Switch to select kind of ECMWF Web Api access and the
305	possible data sets. Public data sets (1) and Memberstate data sets (0).
306	Default value is 0.
307
308	ec_api : boolean
309	Tells wether the ECMWF Web API was able to load or not.
310	Default value is None.
311
312	cds_api : boolean
313	Tells wether the CDS API was able to load or not.
314	Default value is None.
315
316	purefc : int
317	Switch to decide wether the job is a pure forecast retrieval or
318	coupled with analysis data. Default value is 0.
319
320	rrint : int
321	Switch to select between old precipitation disaggregation method (0)
322	or the new IA3 disaggegration method (1). Default value is 0.
323
324	doubleelda : int
325	Switch to select the calculation of extra ensemble members for the
326	ELDA stream. It doubles the amount of retrieved ensemble members.
327
328	logicals : list of str
329	List of the names of logical switches which controls the flow
330	of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
331	'etadiff', 'dpdeta', 'cwc', 'wrf', 'ecstorage',
332	'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint', 'doubleelda']
333	'''
334
335	def __init__(self, filename):
336	'''Initialises the instance of ControlFile class and defines
337	all class attributes with default values. Afterwards calls
338	function __read_controlfile__ to read parameter from Control file.
339
340	Parameters
341	----------
342	filename : str
343	Name of CONTROL file.
344
345	Return
346	------
347
348	'''
349
350	# list of all possible class attributes and their default values
351	self.controlfile = filename
352	self.start_date = None
353	self.end_date = None
354	self.date_chunk = 3
355	self.job_chunk = None
356	self.dtime = None
357	self.basetime = None
358	self.maxstep = None
359	self.type = None
360	self.time = None
361	self.step = None
362	self.acctype = None
363	self.acctime = None
364	self.accmaxstep = None
365	self.marsclass = None
366	self.dataset = None
367	self.stream = None
368	self.number = 'OFF'
369	self.expver = '1'
370	self.gaussian = ''
371	self.grid = None
372	self.area = ''
373	self.left = None
374	self.lower = None
375	self.upper = None
376	self.right = None
377	self.level = None
378	self.levelist = None
379	self.resol = None
380	self.gauss = 0
381	self.accuracy = 24
382	self.omega = 0
383	self.omegadiff = 0
384	self.eta = 0
385	self.etadiff = 0
386	self.etapar = 77
387	self.dpdeta = 1
388	self.smooth = 0
389	self.format = 'GRIB1'
390	self.addpar = None
391	self.prefix = 'EN'
392	self.cwc = 0
393	self.wrf = 0
394	self.ecfsdir = 'ectmp:/${USER}/econdemand/'
395	self.mailfail = ['${USER}']
396	self.mailops = ['${USER}']
397	self.ecstorage = 0
398	self.ectrans = 0
399	self.inputdir = _config.PATH_INPUT_DIR
400	self.outputdir = None
401	self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR
402	self.exedir = _config.PATH_FORTRAN_SRC
403	self.installdir = None
404	self.makefile = 'Makefile.gfortran'
405	self.destination = None
406	self.gateway = None
407	self.ecuid = None
408	self.ecgid = None
409	self.install_target = None
410	self.debug = 0
411	self.oper = 0
412	self.request = 0
413	self.public = 0
414	self.ec_api = None
415	self.cds_api = None
416	self.purefc = 0
417	self.rrint = 0
418	self.doubleelda = 0
419
420	self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
421	'dpdeta', 'cwc', 'wrf', 'ecstorage',
422	'ectrans', 'debug', 'oper', 'request', 'public',
423	'purefc', 'rrint', 'doubleelda']
424
425	self._read_controlfile()
426
427	return
428
429	def _read_controlfile(self):
430	'''Read CONTROL file and assign all CONTROL file variables.
431
432	Parameters
433	----------
434
435	Return
436	------
437
438	'''
439
440	try:
441	cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
442	with open(cfile) as f:
443	fdata = f.read().split('\n')
444	except IOError:
445	print('Could not read CONTROL file "' + cfile + '"')
446	print('Either it does not exist or its syntax is wrong.')
447	print('Try "' + sys.argv[0].split('/')[-1] + \
448	' -h" to print usage information')
449	sys.exit(1)
450
451	# go through every line and store parameter
452	for ldata in fdata:
453	if ldata and ldata[0] == '#':
454	# ignore comment line in control file
455	continue
456	if '#' in ldata:
457	# cut off comment
458	ldata = ldata.split('#')[0]
459	data = ldata.split()
460	if len(data) > 1:
461	if 'm_' in data[0].lower():
462	data[0] = data[0][2:]
463	if data[0].lower() == 'class':
464	data[0] = 'marsclass'
465	if data[0].lower() == 'day1':
466	data[0] = 'start_date'
467	if data[0].lower() == 'day2':
468	data[0] = 'end_date'
469	if len(data) == 2:
470	if '$' in data[1]:
471	setattr(self, data[0].lower(), data[1])
472	while '$' in data[1]:
473	i = data[1].index('$')
474	j = data[1].find('{')
475	k = data[1].find('}')
476	var = os.getenv(data[1][j+1:k])
477	if var is not None:
478	data[1] = data[1][:i] + var + data[1][k+1:]
479	else:
480	my_error('Could not find variable '
481	+ data[1][j+1:k] + ' while reading ' +
482	self.controlfile)
483	setattr(self, data[0].lower() + '_expanded', data[1])
484	else:
485	if data[1].lower() != 'none':
486	setattr(self, data[0].lower(), data[1])
487	else:
488	setattr(self, data[0].lower(), None)
489	elif len(data) > 2:
490	setattr(self, data[0].lower(), (data[1:]))
491	else:
492	pass
493
494	return
495
496	def __str__(self):
497	'''Prepares a string which have all the ControlFile class attributes
498	with its associated values. Each attribute is printed in one line and
499	in alphabetical order.
500
501	Example
502	-------
503	'age': 10
504	'color': 'Spotted'
505	'kids': 0
506	'legs': 2
507	'name': 'Dog'
508	'smell': 'Alot'
509
510	Parameters
511	----------
512
513	Return
514	------
515	string
516	Single string of concatenated ControlFile class attributes
517	with their values
518	'''
519	import collections
520
521	attrs = vars(self).copy()
522	attrs = collections.OrderedDict(sorted(attrs.items()))
523
524	return '\n'.join("%s: %s" % item for item in attrs.items())
525
526	def assign_args_to_control(self, args):
527	'''Overwrites the existing ControlFile instance attributes with
528	the command line arguments.
529
530	Parameters
531	----------
532	args : Namespace
533	Contains the commandline arguments from script/program call.
534
535	Return
536	------
537
538	'''
539
540	# get dictionary of command line parameters and eliminate all
541	# parameters which are None (were not specified)
542	args_dict = vars(args)
543	arguments = {k : args_dict[k] for k in args_dict
544	if args_dict[k] != None}
545
546	# assign all passed command line arguments to ControlFile instance
547	for k, v in arguments.items():
548	setattr(self, str(k), v)
549
550	return
551
552	def assign_envs_to_control(self, envs):
553	'''Assigns the ECMWF environment parameter.
554
555	Parameters
556	----------
557	envs : dict of str
558	Contains the ECMWF environment parameternames "ECUID", "ECGID",
559	"DESTINATION" and "GATEWAY" with its corresponding values.
560	They were read from the file "ECMWF_ENV".
561
562	Return
563	------
564
565	'''
566
567	for k, v in envs.items():
568	setattr(self, str(k).lower(), str(v))
569
570	return
571
572	def check_conditions(self, queue):
573	'''Checks a couple of necessary attributes and conditions,
574	such as if they exist and contain values.
575	Otherwise set default values.
576
577	Parameters
578	----------
579	queue : str
580	Name of the queue if submitted to the ECMWF servers.
581	Used to check if ecuid, ecgid, gateway and destination
582	are set correctly and are not empty.
583
584	Return
585	------
586
587	'''
588	check_logicals_type(self, self.logicals)
589
590	self.mailfail = check_mail(self.mailfail)
591
592	self.mailops = check_mail(self.mailops)
593
594	check_queue(queue, self.gateway, self.destination,
595	self.ecuid, self.ecgid)
596
597	self.outputdir, self.installdir = check_pathes(self.inputdir,
598	self.outputdir,
599	self.installdir,
600	self.flexextractdir)
601
602	self.start_date, self.end_date = check_dates(self.start_date,
603	self.end_date)
604
605	self.basetime = check_basetime(self.basetime)
606
607	self.levelist, self.level = check_levels(self.levelist, self.level)
608
609	self.step = check_step(self.step)
610
611	self.maxstep = check_maxstep(self.maxstep, self.step)
612
613	check_request(self.request,
614	os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
615
616	check_public(self.public, self.dataset)
617
618	self.type = check_type(self.type, self.step)
619
620	self.time = check_time(self.time)
621
622	self.purefc = check_purefc(self.type)
623
624	self.type, self.time, self.step = check_len_type_time_step(self.type,
625	self.time,
626	self.step,
627	self.maxstep,
628	self.purefc)
629
630	self.acctype = check_acctype(self.acctype, self.type)
631
632	self.acctime = check_acctime(self.acctime, self.marsclass,
633	self.purefc, self.time)
634
635	self.accmaxstep = check_accmaxstep(self.accmaxstep, self.marsclass,
636	self.purefc, self.maxstep)
637
638	self.grid = check_grid(self.grid)
639
640	self.area = check_area(self.grid, self.area, self.upper, self.lower,
641	self.left, self.right)
642
643	self.addpar = check_addpar(self.addpar)
644
645	self.job_chunk = check_job_chunk(self.job_chunk)
646
647	self.number = check_number(self.number)
648
649	return
650
651	def to_list(self):
652	'''Just generates a list of strings containing the attributes and
653	assigned values except the attributes "_expanded", "exedir",
654	"flexextractdir" and "installdir".
655
656	Parameters
657	----------
658
659	Return
660	------
661	l : list of *
662	A sorted list of the all ControlFile class attributes with
663	their values except the attributes "_expanded", "exedir",
664	"flexextractdir" and "installdir".
665	'''
666
667	import collections
668
669	attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
670
671	l = list()
672
673	for item in attrs.items():
674	if '_expanded' in item[0]:
675	pass
676	elif 'exedir' in item[0]:
677	pass
678	elif 'installdir' in item[0]:
679	pass
680	elif 'flexextractdir' in item[0]:
681	pass
682	else:
683	if isinstance(item[1], list):
684	stot = ''
685	for s in item[1]:
686	stot += s + ' '
687
688	l.append("%s %s\n" % (item[0], stot))
689	else:
690	l.append("%s %s\n" % item)
691
692	return sorted(l)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: