Context Navigation

source: flex_extract.git/Source/Python/Classes/ControlFile.py @ 75db9b0

dev

Last change on this file since 75db9b0 was 75db9b0, checked in by anphi <anne.philipp@…>, 4 years ago
added history information of changes to v7.1.2
Property mode set to `100644`
File size: 22.9 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - applied some minor modifications in programming style/structure
14	# - changed name of class Control to ControlFile for more
15	# self-explanation naming
16	# - outsource of class ControlFile
17	# - initialisation of class attributes ( to avoid high number of
18	# conditional statements and set default values )
19	# - divided assignment of attributes and the check of conditions
20	# - outsourced the commandline argument assignments to control attributes
21	# June 2020 - Anne Philipp
22	# - update default makefile to None
23	# August 2020 - Leopold Haimberger
24	# - added a class parameter for system installation path
25	#
26	# @License:
27	# (C) Copyright 2014-2020.
28	# Anne Philipp, Leopold Haimberger
29	#
30	# SPDX-License-Identifier: CC-BY-4.0
31	#
32	# This work is licensed under the Creative Commons Attribution 4.0
33	# International License. To view a copy of this license, visit
34	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
35	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
36	#*******************************************************************************
37
38	# ------------------------------------------------------------------------------
39	# MODULES
40	# ------------------------------------------------------------------------------
41	from __future__ import print_function
42
43	import os
44	import sys
45
46	# software specific classes and modules from flex_extract
47	#pylint: disable=wrong-import-position
48	sys.path.append('../')
49	import _config
50	from Mods.tools import my_error
51	from Mods.checks import (check_grid, check_area, check_levels, check_purefc,
52	check_step, check_mail, check_queue, check_pathes,
53	check_dates, check_maxstep, check_type, check_request,
54	check_basetime, check_public, check_acctype,
55	check_acctime, check_accmaxstep, check_time,
56	check_logicals_type, check_len_type_time_step,
57	check_addpar, check_job_chunk, check_number)
58	#pylint: enable=wrong-import-position
59
60	# ------------------------------------------------------------------------------
61	# CLASS
62	# ------------------------------------------------------------------------------
63	class ControlFile(object):
64	'''
65	Contains the information which are stored in the CONTROL files.
66
67	The CONTROL file is the steering part of the FLEXPART extraction
68	software. All necessary parameters needed to retrieve the data fields
69	from the MARS archive for driving FLEXPART are set in a CONTROL file.
70	Some specific parameters like the start and end dates can be overwritten
71	by the command line parameters, but in generall all parameters needed
72	for a complete set of fields for FLEXPART can be set in the CONTROL file.
73
74	Attributes
75	----------
76	controlfile : str
77	The name of the control file to be processed. Default value is the
78	filename passed to the init function when initialised.
79
80	start_date : str
81	The first day of the retrieval period. Default value is None.
82
83	end_date :str
84	The last day of the retrieval period. Default value is None.
85
86	date_chunk : int
87	Length of period for a single mars retrieval. Default value is 3.
88
89	dtime :str
90	The time step in hours. Default value is None.
91
92	basetime : int
93	The time for a half day retrieval. The 12 hours upfront are to be
94	retrieved. Default value is None.
95
96	maxstep : int
97	The maximum forecast step for non flux data. Default value is None.
98
99	type : list of str
100	List of field type per retrieving hour. Default value is None.
101
102	time : list of str
103	List of retrieving times in hours. Default valuer is None.
104
105	step : list of str or str
106	List of forecast time steps in hours for non flux data.
107	Default value is None.
108
109	acctype : str
110	The field type for the accumulated forecast fields.
111	Default value is None.
112
113	acctime : str
114	The starting time of the accumulated forecasts. Default value is None.
115
116	accmaxstep : int
117	The maximum forecast step for the accumulated forecast fields
118	(flux data). Default value is None.
119
120	marsclass : str
121	Characterisation of dataset. Default value is None.
122
123	dataset : str
124	For public datasets there is the specific naming and parameter
125	dataset which has to be used to characterize the type of
126	data. Default value is None.
127
128	stream : str
129	Identifies the forecasting system used to generate the data.
130	Default value is None.
131
132	number : str
133	Selects the member in ensemble forecast run. Default value is 'OFF'.
134
135	expver : str
136	The version number of the dataset. Default value is '1'.
137
138	gaussian : str
139	This parameter is deprecated and should no longer be used.
140	Specifies the desired type of Gaussian grid for the output.
141	Default value is an empty string ''.
142
143	grid : str
144	Specifies the output grid which can be either a Gaussian grid
145	or a Latitude/Longitude grid. Default value is None.
146
147	area : str
148	Specifies the desired sub-area of data to be extracted.
149	Default value is None.
150
151	left : str
152	The western most longitude of the area to be extracted.
153	Default value is None.
154
155	lower : str
156	The southern most latitude of the area to be extracted.
157	Default value is None.
158
159	upper : str
160	The northern most latitued of the area to be extracted.
161	Default value is None.
162
163	right : str
164	The eastern most longitude of the area to be extracted.
165	Default value is None.
166
167	level : str
168	Specifies the maximum level. Default value is None.
169
170	levelist : str
171	Specifies the required level list. Default value is None.
172
173	resol : str
174	Specifies the desired triangular truncation of retrieved data,
175	before carrying out any other selected post-processing.
176	Default value is None.
177
178	gauss : int
179	Switch to select gaussian fields (1) or regular lat/lon (0).
180	Default value is 0.
181
182	accuracy : int
183	Specifies the number of bits per value to be used in the
184	generated GRIB coded fields. Default value is 24.
185
186	omega : int
187	Switch to select omega retrieval (1) or not (0). Default value is 0.
188
189	omegadiff : int
190	Switch to decide to calculate Omega and Dps/Dt from continuity
191	equation for diagnostic purposes (1) or not (0). Default value is 0.
192
193	eta : int
194	Switch to select direct retrieval of etadot from MARS (1) or
195	wether it has to be calculated (0). Then Default value is 0.
196
197	etadiff : int
198	Switch to select calculation of etadot and Dps/Dt from continuity
199	equation for diagnostic purposes (1) or not (0). Default value is 0.
200
201	etapar : int
202	GRIB parameter Id for etadot fields. Default value is 77.
203
204	dpdeta : int
205	Switch to select multiplication of etadot with dpdeta.
206	Default value is 1.
207
208	smooth : int
209	Spectral truncation of ETADOT after calculation on Gaussian grid.
210	Default value is 0.
211
212	format : str
213	The format of the GRIB data. Default value is 'GRIB1'.
214
215	addpar : str
216	List of additional surface level ECMWF parameter to be retrieved.
217	Default value is None.
218
219	prefix : str
220	Prefix string for the final FLEXPART/FLEXTRA ready input files.
221	Default value is 'EN'.
222
223	cwc : int
224	Switch to select wether the sum of cloud liquid water content and
225	cloud ice water content should be retrieved. Default value is 0.
226
227	wrf : int
228	Switch to select further parameters for retrievment to support
229	WRF simulations. Default value is 0.
230
231	ecfsdir : str
232	Path to the ECMWF storage 'ectmp:/${USER}/econdemand/'
233
234	mailfail : list of str
235	Email list for sending error log files from ECMWF servers.
236	The email addresses should be seperated by a comma.
237	Default value is ['${USER}'].
238
239	mailops : list of str
240	Email list for sending operational log files from ECMWF servers.
241	The email addresses should be seperated by a comma.
242	Default value is ['${USER}'].
243
244	ecstorage : int
245	Switch to select storage of FLEXPART ready output files
246	in the ECFS file system. Default value is 0.
247
248	ectrans : int
249	Switch to select the transfer of FLEXPART ready output files
250	to the gateway server. Default value is 0.
251
252	inputdir : str
253	Path to the temporary directory for the retrieval grib files and
254	other processing files. Default value is _config.PATH_INPUT_DIR.
255
256	outputdir : str
257	Path to the final directory where the final FLEXPART ready input
258	files are stored. Default value is None.
259
260	flexextractdir : str
261	Path to the flex_extract root directory. Default value is
262	_config.PATH_FLEXEXTRACT_DIR.
263
264	exedir : str
265	Path to the FORTRAN executable file. Default value is
266	_config.PATH_FORTRAN_SRC.
267
268	installdir : str
269	Path to a FLEXPART root directory. Default value is None.
270
271	makefile : str
272	Name of the makefile to be used for the Fortran program.
273	Default value is None.
274
275	destination : str
276	The remote destination which is used to transfer files
277	from ECMWF server to local gateway server. Default value is None.
278
279	gateway : str
280	The gateway server the user is using. Default value is None.
281
282	ecuid : str
283	The user id on ECMWF server. Default value is None.
284
285	ecgid : str
286	The group id on ECMWF server. Default value is None.
287
288	install_target : str
289	Defines the location where the installation is to be done.
290	Default value is None.
291
292	debug : int
293	Switch to keep temporary files at the end of postprocessing (1) or
294	to delete all temporary files except the final output files (0).
295	Default value is 0.
296
297	oper : int
298	Switch to prepare the operational job script. Start date, end date and
299	basetime will be prepared with environment variables.
300	Default value is 0.
301
302	request : int
303	Switch to select between just retrieving the data (0), writing the mars
304	parameter values to a csv file (1) or doing both (2).
305	Default value is 0.
306
307	public : int
308	Switch to select kind of ECMWF Web Api access and the
309	possible data sets. Public data sets (1) and Memberstate data sets (0).
310	Default value is 0.
311
312	ec_api : boolean
313	Tells wether the ECMWF Web API was able to load or not.
314	Default value is None.
315
316	cds_api : boolean
317	Tells wether the CDS API was able to load or not.
318	Default value is None.
319
320	purefc : int
321	Switch to decide wether the job is a pure forecast retrieval or
322	coupled with analysis data. Default value is 0.
323
324	rrint : int
325	Switch to select between old precipitation disaggregation method (0)
326	or the new IA3 disaggegration method (1). Default value is 0.
327
328	doubleelda : int
329	Switch to select the calculation of extra ensemble members for the
330	ELDA stream. It doubles the amount of retrieved ensemble members.
331
332	logicals : list of str
333	List of the names of logical switches which controls the flow
334	of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
335	'etadiff', 'dpdeta', 'cwc', 'wrf', 'ecstorage',
336	'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint', 'doubleelda']
337	'''
338
339	def __init__(self, filename):
340	'''Initialises the instance of ControlFile class and defines
341	all class attributes with default values. Afterwards calls
342	function __read_controlfile__ to read parameter from Control file.
343
344	Parameters
345	----------
346	filename : str
347	Name of CONTROL file.
348
349	Return
350	------
351
352	'''
353
354	# list of all possible class attributes and their default values
355	self.controlfile = filename
356	self.start_date = None
357	self.end_date = None
358	self.date_chunk = 3
359	self.job_chunk = None
360	self.dtime = None
361	self.basetime = None
362	self.maxstep = None
363	self.type = None
364	self.time = None
365	self.step = None
366	self.acctype = None
367	self.acctime = None
368	self.accmaxstep = None
369	self.marsclass = None
370	self.dataset = None
371	self.stream = None
372	self.number = 'OFF'
373	self.expver = '1'
374	self.gaussian = ''
375	self.grid = None
376	self.area = ''
377	self.left = None
378	self.lower = None
379	self.upper = None
380	self.right = None
381	self.level = None
382	self.levelist = None
383	self.resol = None
384	self.gauss = 0
385	self.accuracy = 24
386	self.omega = 0
387	self.omegadiff = 0
388	self.eta = 0
389	self.etadiff = 0
390	self.etapar = 77
391	self.dpdeta = 1
392	self.smooth = 0
393	self.format = 'GRIB1'
394	self.addpar = None
395	self.prefix = 'EN'
396	self.cwc = 0
397	self.wrf = 0
398	self.ecfsdir = 'ectmp:/${USER}/econdemand/'
399	self.mailfail = ['${USER}']
400	self.mailops = ['${USER}']
401	self.ecstorage = 0
402	self.ectrans = 0
403	self.inputdir = _config.PATH_INPUT_DIR
404	self.outputdir = None
405	self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR
406	self.exedir = _config.PATH_FORTRAN_SRC
407	self.installdir = None
408	self.sysinstalldir = None
409	self.makefile = None
410	self.destination = None
411	self.gateway = None
412	self.ecuid = None
413	self.ecgid = None
414	self.install_target = None
415	self.debug = 0
416	self.oper = 0
417	self.request = 0
418	self.public = 0
419	self.ec_api = None
420	self.cds_api = None
421	self.purefc = 0
422	self.rrint = 0
423	self.doubleelda = 0
424
425	self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
426	'dpdeta', 'cwc', 'wrf', 'ecstorage',
427	'ectrans', 'debug', 'oper', 'request', 'public',
428	'purefc', 'rrint', 'doubleelda']
429
430	self._read_controlfile()
431
432	return
433
434	def _read_controlfile(self):
435	'''Read CONTROL file and assign all CONTROL file variables.
436
437	Parameters
438	----------
439
440	Return
441	------
442
443	'''
444
445	try:
446	cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
447	with open(cfile) as f:
448	fdata = f.read().split('\n')
449	except IOError:
450	print('Could not read CONTROL file "' + cfile + '"')
451	print('Either it does not exist or its syntax is wrong.')
452	print('Try "' + sys.argv[0].split('/')[-1] + \
453	' -h" to print usage information')
454	sys.exit(1)
455
456	# go through every line and store parameter
457	for ldata in fdata:
458	if ldata and ldata[0] == '#':
459	# ignore comment line in control file
460	continue
461	if '#' in ldata:
462	# cut off comment
463	ldata = ldata.split('#')[0]
464	data = ldata.split()
465	if len(data) > 1:
466	if 'm_' in data[0].lower():
467	data[0] = data[0][2:]
468	if data[0].lower() == 'class':
469	data[0] = 'marsclass'
470	if data[0].lower() == 'day1':
471	data[0] = 'start_date'
472	if data[0].lower() == 'day2':
473	data[0] = 'end_date'
474	if len(data) == 2:
475	if '$' in data[1]:
476	setattr(self, data[0].lower(), data[1])
477	while '$' in data[1]:
478	i = data[1].index('$')
479	j = data[1].find('{')
480	k = data[1].find('}')
481	var = os.getenv(data[1][j+1:k])
482	if var is not None:
483	data[1] = data[1][:i] + var + data[1][k+1:]
484	else:
485	my_error('Could not find variable '
486	+ data[1][j+1:k] + ' while reading ' +
487	self.controlfile)
488	setattr(self, data[0].lower() + '_expanded', data[1])
489	else:
490	if data[1].lower() != 'none':
491	setattr(self, data[0].lower(), data[1])
492	else:
493	setattr(self, data[0].lower(), None)
494	elif len(data) > 2:
495	setattr(self, data[0].lower(), (data[1:]))
496	else:
497	pass
498
499	return
500
501	def __str__(self):
502	'''Prepares a string which have all the ControlFile class attributes
503	with its associated values. Each attribute is printed in one line and
504	in alphabetical order.
505
506	Example
507	-------
508	'age': 10
509	'color': 'Spotted'
510	'kids': 0
511	'legs': 2
512	'name': 'Dog'
513	'smell': 'Alot'
514
515	Parameters
516	----------
517
518	Return
519	------
520	string
521	Single string of concatenated ControlFile class attributes
522	with their values
523	'''
524	import collections
525
526	attrs = vars(self).copy()
527	attrs = collections.OrderedDict(sorted(attrs.items()))
528
529	return '\n'.join("%s: %s" % item for item in attrs.items())
530
531	def assign_args_to_control(self, args):
532	'''Overwrites the existing ControlFile instance attributes with
533	the command line arguments.
534
535	Parameters
536	----------
537	args : Namespace
538	Contains the commandline arguments from script/program call.
539
540	Return
541	------
542
543	'''
544
545	# get dictionary of command line parameters and eliminate all
546	# parameters which are None (were not specified)
547	args_dict = vars(args)
548	arguments = {k : args_dict[k] for k in args_dict
549	if args_dict[k] != None}
550
551	# assign all passed command line arguments to ControlFile instance
552	for k, v in arguments.items():
553	setattr(self, str(k), v)
554
555	return
556
557	def assign_envs_to_control(self, envs):
558	'''Assigns the ECMWF environment parameter.
559
560	Parameters
561	----------
562	envs : dict of str
563	Contains the ECMWF environment parameternames "ECUID", "ECGID",
564	"DESTINATION" and "GATEWAY" with its corresponding values.
565	They were read from the file "ECMWF_ENV".
566
567	Return
568	------
569
570	'''
571
572	for k, v in envs.items():
573	setattr(self, str(k).lower(), str(v))
574
575	return
576
577	def check_conditions(self, queue):
578	'''Checks a couple of necessary attributes and conditions,
579	such as if they exist and contain values.
580	Otherwise set default values.
581
582	Parameters
583	----------
584	queue : str
585	Name of the queue if submitted to the ECMWF servers.
586	Used to check if ecuid, ecgid, gateway and destination
587	are set correctly and are not empty.
588
589	Return
590	------
591
592	'''
593	check_logicals_type(self, self.logicals)
594
595	self.mailfail = check_mail(self.mailfail)
596
597	self.mailops = check_mail(self.mailops)
598
599	check_queue(queue, self.gateway, self.destination,
600	self.ecuid, self.ecgid)
601
602	self.outputdir, self.installdir = check_pathes(self.inputdir,
603	self.outputdir,
604	self.installdir,
605	self.flexextractdir)
606
607	self.start_date, self.end_date = check_dates(self.start_date,
608	self.end_date)
609
610	self.basetime = check_basetime(self.basetime)
611
612	self.levelist, self.level = check_levels(self.levelist, self.level)
613
614	self.step = check_step(self.step)
615
616	self.maxstep = check_maxstep(self.maxstep, self.step)
617
618	check_request(self.request,
619	os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
620
621	check_public(self.public, self.dataset, self.marsclass)
622
623	self.type = check_type(self.type, self.step)
624
625	self.time = check_time(self.time)
626
627	self.purefc = check_purefc(self.type)
628
629	self.type, self.time, self.step = check_len_type_time_step(self.type,
630	self.time,
631	self.step,
632	self.maxstep,
633	self.purefc)
634
635	self.acctype = check_acctype(self.acctype, self.type)
636
637	self.acctime = check_acctime(self.acctime, self.marsclass,
638	self.purefc, self.time)
639
640	self.accmaxstep = check_accmaxstep(self.accmaxstep, self.marsclass,
641	self.purefc, self.maxstep)
642
643	self.grid = check_grid(self.grid)
644
645	self.area = check_area(self.grid, self.area, self.upper, self.lower,
646	self.left, self.right)
647
648	self.addpar = check_addpar(self.addpar)
649
650	self.job_chunk = check_job_chunk(self.job_chunk)
651
652	self.number = check_number(self.number)
653
654	return
655
656	def to_list(self):
657	'''Just generates a list of strings containing the attributes and
658	assigned values except the attributes "_expanded", "exedir",
659	"flexextractdir" and "installdir".
660
661	Parameters
662	----------
663
664	Return
665	------
666	l : list of *
667	A sorted list of the all ControlFile class attributes with
668	their values except the attributes "_expanded", "exedir",
669	"flexextractdir" and "installdir".
670	'''
671
672	import collections
673
674	attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
675
676	l = list()
677
678	for item in attrs.items():
679	if '_expanded' in item[0]:
680	pass
681	elif 'exedir' in item[0]:
682	pass
683	elif 'installdir' in item[0]:
684	pass
685	elif 'flexextractdir' in item[0]:
686	pass
687	else:
688	if isinstance(item[1], list):
689	stot = ''
690	for s in item[1]:
691	stot += s + ' '
692
693	l.append("%s %s\n" % (item[0], stot))
694	else:
695	l.append("%s %s\n" % item)
696
697	return sorted(l)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: