Context Navigation

source: flex_extract.git/Source/Python/Classes/ControlFile.py @ 47be2684

ctbtodev

Last change on this file since 47be2684 was 47be2684, checked in by Leopold Haimberger <leopold.haimberger@…>, 4 years ago
Adaptations to allow for a system installation with separate user and system path. Updated documentation
Property mode set to `100644`
File size: 22.8 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - applied some minor modifications in programming style/structure
14	# - changed name of class Control to ControlFile for more
15	# self-explanation naming
16	# - outsource of class ControlFile
17	# - initialisation of class attributes ( to avoid high number of
18	# conditional statements and set default values )
19	# - divided assignment of attributes and the check of conditions
20	# - outsourced the commandline argument assignments to control attributes
21	# June 2020 - Anne Philipp
22	# - update default makefile to None
23	#
24	# @License:
25	# (C) Copyright 2014-2020.
26	# Anne Philipp, Leopold Haimberger
27	#
28	# SPDX-License-Identifier: CC-BY-4.0
29	#
30	# This work is licensed under the Creative Commons Attribution 4.0
31	# International License. To view a copy of this license, visit
32	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
33	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
34	#*******************************************************************************
35
36	# ------------------------------------------------------------------------------
37	# MODULES
38	# ------------------------------------------------------------------------------
39	from __future__ import print_function
40
41	import os
42	import sys
43
44	# software specific classes and modules from flex_extract
45	#pylint: disable=wrong-import-position
46	sys.path.append('../')
47	import _config
48	from Mods.tools import my_error
49	from Mods.checks import (check_grid, check_area, check_levels, check_purefc,
50	check_step, check_mail, check_queue, check_pathes,
51	check_dates, check_maxstep, check_type, check_request,
52	check_basetime, check_public, check_acctype,
53	check_acctime, check_accmaxstep, check_time,
54	check_logicals_type, check_len_type_time_step,
55	check_addpar, check_job_chunk, check_number)
56	#pylint: enable=wrong-import-position
57
58	# ------------------------------------------------------------------------------
59	# CLASS
60	# ------------------------------------------------------------------------------
61	class ControlFile(object):
62	'''
63	Contains the information which are stored in the CONTROL files.
64
65	The CONTROL file is the steering part of the FLEXPART extraction
66	software. All necessary parameters needed to retrieve the data fields
67	from the MARS archive for driving FLEXPART are set in a CONTROL file.
68	Some specific parameters like the start and end dates can be overwritten
69	by the command line parameters, but in generall all parameters needed
70	for a complete set of fields for FLEXPART can be set in the CONTROL file.
71
72	Attributes
73	----------
74	controlfile : str
75	The name of the control file to be processed. Default value is the
76	filename passed to the init function when initialised.
77
78	start_date : str
79	The first day of the retrieval period. Default value is None.
80
81	end_date :str
82	The last day of the retrieval period. Default value is None.
83
84	date_chunk : int
85	Length of period for a single mars retrieval. Default value is 3.
86
87	dtime :str
88	The time step in hours. Default value is None.
89
90	basetime : int
91	The time for a half day retrieval. The 12 hours upfront are to be
92	retrieved. Default value is None.
93
94	maxstep : int
95	The maximum forecast step for non flux data. Default value is None.
96
97	type : list of str
98	List of field type per retrieving hour. Default value is None.
99
100	time : list of str
101	List of retrieving times in hours. Default valuer is None.
102
103	step : list of str or str
104	List of forecast time steps in hours for non flux data.
105	Default value is None.
106
107	acctype : str
108	The field type for the accumulated forecast fields.
109	Default value is None.
110
111	acctime : str
112	The starting time of the accumulated forecasts. Default value is None.
113
114	accmaxstep : int
115	The maximum forecast step for the accumulated forecast fields
116	(flux data). Default value is None.
117
118	marsclass : str
119	Characterisation of dataset. Default value is None.
120
121	dataset : str
122	For public datasets there is the specific naming and parameter
123	dataset which has to be used to characterize the type of
124	data. Default value is None.
125
126	stream : str
127	Identifies the forecasting system used to generate the data.
128	Default value is None.
129
130	number : str
131	Selects the member in ensemble forecast run. Default value is 'OFF'.
132
133	expver : str
134	The version number of the dataset. Default value is '1'.
135
136	gaussian : str
137	This parameter is deprecated and should no longer be used.
138	Specifies the desired type of Gaussian grid for the output.
139	Default value is an empty string ''.
140
141	grid : str
142	Specifies the output grid which can be either a Gaussian grid
143	or a Latitude/Longitude grid. Default value is None.
144
145	area : str
146	Specifies the desired sub-area of data to be extracted.
147	Default value is None.
148
149	left : str
150	The western most longitude of the area to be extracted.
151	Default value is None.
152
153	lower : str
154	The southern most latitude of the area to be extracted.
155	Default value is None.
156
157	upper : str
158	The northern most latitued of the area to be extracted.
159	Default value is None.
160
161	right : str
162	The eastern most longitude of the area to be extracted.
163	Default value is None.
164
165	level : str
166	Specifies the maximum level. Default value is None.
167
168	levelist : str
169	Specifies the required level list. Default value is None.
170
171	resol : str
172	Specifies the desired triangular truncation of retrieved data,
173	before carrying out any other selected post-processing.
174	Default value is None.
175
176	gauss : int
177	Switch to select gaussian fields (1) or regular lat/lon (0).
178	Default value is 0.
179
180	accuracy : int
181	Specifies the number of bits per value to be used in the
182	generated GRIB coded fields. Default value is 24.
183
184	omega : int
185	Switch to select omega retrieval (1) or not (0). Default value is 0.
186
187	omegadiff : int
188	Switch to decide to calculate Omega and Dps/Dt from continuity
189	equation for diagnostic purposes (1) or not (0). Default value is 0.
190
191	eta : int
192	Switch to select direct retrieval of etadot from MARS (1) or
193	wether it has to be calculated (0). Then Default value is 0.
194
195	etadiff : int
196	Switch to select calculation of etadot and Dps/Dt from continuity
197	equation for diagnostic purposes (1) or not (0). Default value is 0.
198
199	etapar : int
200	GRIB parameter Id for etadot fields. Default value is 77.
201
202	dpdeta : int
203	Switch to select multiplication of etadot with dpdeta.
204	Default value is 1.
205
206	smooth : int
207	Spectral truncation of ETADOT after calculation on Gaussian grid.
208	Default value is 0.
209
210	format : str
211	The format of the GRIB data. Default value is 'GRIB1'.
212
213	addpar : str
214	List of additional surface level ECMWF parameter to be retrieved.
215	Default value is None.
216
217	prefix : str
218	Prefix string for the final FLEXPART/FLEXTRA ready input files.
219	Default value is 'EN'.
220
221	cwc : int
222	Switch to select wether the sum of cloud liquid water content and
223	cloud ice water content should be retrieved. Default value is 0.
224
225	wrf : int
226	Switch to select further parameters for retrievment to support
227	WRF simulations. Default value is 0.
228
229	ecfsdir : str
230	Path to the ECMWF storage 'ectmp:/${USER}/econdemand/'
231
232	mailfail : list of str
233	Email list for sending error log files from ECMWF servers.
234	The email addresses should be seperated by a comma.
235	Default value is ['${USER}'].
236
237	mailops : list of str
238	Email list for sending operational log files from ECMWF servers.
239	The email addresses should be seperated by a comma.
240	Default value is ['${USER}'].
241
242	ecstorage : int
243	Switch to select storage of FLEXPART ready output files
244	in the ECFS file system. Default value is 0.
245
246	ectrans : int
247	Switch to select the transfer of FLEXPART ready output files
248	to the gateway server. Default value is 0.
249
250	inputdir : str
251	Path to the temporary directory for the retrieval grib files and
252	other processing files. Default value is _config.PATH_INPUT_DIR.
253
254	outputdir : str
255	Path to the final directory where the final FLEXPART ready input
256	files are stored. Default value is None.
257
258	flexextractdir : str
259	Path to the flex_extract root directory. Default value is
260	_config.PATH_FLEXEXTRACT_DIR.
261
262	exedir : str
263	Path to the FORTRAN executable file. Default value is
264	_config.PATH_FORTRAN_SRC.
265
266	installdir : str
267	Path to a FLEXPART root directory. Default value is None.
268
269	makefile : str
270	Name of the makefile to be used for the Fortran program.
271	Default value is None.
272
273	destination : str
274	The remote destination which is used to transfer files
275	from ECMWF server to local gateway server. Default value is None.
276
277	gateway : str
278	The gateway server the user is using. Default value is None.
279
280	ecuid : str
281	The user id on ECMWF server. Default value is None.
282
283	ecgid : str
284	The group id on ECMWF server. Default value is None.
285
286	install_target : str
287	Defines the location where the installation is to be done.
288	Default value is None.
289
290	debug : int
291	Switch to keep temporary files at the end of postprocessing (1) or
292	to delete all temporary files except the final output files (0).
293	Default value is 0.
294
295	oper : int
296	Switch to prepare the operational job script. Start date, end date and
297	basetime will be prepared with environment variables.
298	Default value is 0.
299
300	request : int
301	Switch to select between just retrieving the data (0), writing the mars
302	parameter values to a csv file (1) or doing both (2).
303	Default value is 0.
304
305	public : int
306	Switch to select kind of ECMWF Web Api access and the
307	possible data sets. Public data sets (1) and Memberstate data sets (0).
308	Default value is 0.
309
310	ec_api : boolean
311	Tells wether the ECMWF Web API was able to load or not.
312	Default value is None.
313
314	cds_api : boolean
315	Tells wether the CDS API was able to load or not.
316	Default value is None.
317
318	purefc : int
319	Switch to decide wether the job is a pure forecast retrieval or
320	coupled with analysis data. Default value is 0.
321
322	rrint : int
323	Switch to select between old precipitation disaggregation method (0)
324	or the new IA3 disaggegration method (1). Default value is 0.
325
326	doubleelda : int
327	Switch to select the calculation of extra ensemble members for the
328	ELDA stream. It doubles the amount of retrieved ensemble members.
329
330	logicals : list of str
331	List of the names of logical switches which controls the flow
332	of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
333	'etadiff', 'dpdeta', 'cwc', 'wrf', 'ecstorage',
334	'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint', 'doubleelda']
335	'''
336
337	def __init__(self, filename):
338	'''Initialises the instance of ControlFile class and defines
339	all class attributes with default values. Afterwards calls
340	function __read_controlfile__ to read parameter from Control file.
341
342	Parameters
343	----------
344	filename : str
345	Name of CONTROL file.
346
347	Return
348	------
349
350	'''
351
352	# list of all possible class attributes and their default values
353	self.controlfile = filename
354	self.start_date = None
355	self.end_date = None
356	self.date_chunk = 3
357	self.job_chunk = None
358	self.dtime = None
359	self.basetime = None
360	self.maxstep = None
361	self.type = None
362	self.time = None
363	self.step = None
364	self.acctype = None
365	self.acctime = None
366	self.accmaxstep = None
367	self.marsclass = None
368	self.dataset = None
369	self.stream = None
370	self.number = 'OFF'
371	self.expver = '1'
372	self.gaussian = ''
373	self.grid = None
374	self.area = ''
375	self.left = None
376	self.lower = None
377	self.upper = None
378	self.right = None
379	self.level = None
380	self.levelist = None
381	self.resol = None
382	self.gauss = 0
383	self.accuracy = 24
384	self.omega = 0
385	self.omegadiff = 0
386	self.eta = 0
387	self.etadiff = 0
388	self.etapar = 77
389	self.dpdeta = 1
390	self.smooth = 0
391	self.format = 'GRIB1'
392	self.addpar = None
393	self.prefix = 'EN'
394	self.cwc = 0
395	self.wrf = 0
396	self.ecfsdir = 'ectmp:/${USER}/econdemand/'
397	self.mailfail = ['${USER}']
398	self.mailops = ['${USER}']
399	self.ecstorage = 0
400	self.ectrans = 0
401	self.inputdir = _config.PATH_INPUT_DIR
402	self.outputdir = None
403	self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR
404	self.exedir = _config.PATH_FORTRAN_SRC
405	self.installdir = None
406	self.sysinstalldir = None
407	self.makefile = None
408	self.destination = None
409	self.gateway = None
410	self.ecuid = None
411	self.ecgid = None
412	self.install_target = None
413	self.debug = 0
414	self.oper = 0
415	self.request = 0
416	self.public = 0
417	self.ec_api = None
418	self.cds_api = None
419	self.purefc = 0
420	self.rrint = 0
421	self.doubleelda = 0
422
423	self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
424	'dpdeta', 'cwc', 'wrf', 'ecstorage',
425	'ectrans', 'debug', 'oper', 'request', 'public',
426	'purefc', 'rrint', 'doubleelda']
427
428	self._read_controlfile()
429
430	return
431
432	def _read_controlfile(self):
433	'''Read CONTROL file and assign all CONTROL file variables.
434
435	Parameters
436	----------
437
438	Return
439	------
440
441	'''
442
443	try:
444	cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
445	with open(cfile) as f:
446	fdata = f.read().split('\n')
447	except IOError:
448	print('Could not read CONTROL file "' + cfile + '"')
449	print('Either it does not exist or its syntax is wrong.')
450	print('Try "' + sys.argv[0].split('/')[-1] + \
451	' -h" to print usage information')
452	sys.exit(1)
453
454	# go through every line and store parameter
455	for ldata in fdata:
456	if ldata and ldata[0] == '#':
457	# ignore comment line in control file
458	continue
459	if '#' in ldata:
460	# cut off comment
461	ldata = ldata.split('#')[0]
462	data = ldata.split()
463	if len(data) > 1:
464	if 'm_' in data[0].lower():
465	data[0] = data[0][2:]
466	if data[0].lower() == 'class':
467	data[0] = 'marsclass'
468	if data[0].lower() == 'day1':
469	data[0] = 'start_date'
470	if data[0].lower() == 'day2':
471	data[0] = 'end_date'
472	if len(data) == 2:
473	if '$' in data[1]:
474	setattr(self, data[0].lower(), data[1])
475	while '$' in data[1]:
476	i = data[1].index('$')
477	j = data[1].find('{')
478	k = data[1].find('}')
479	var = os.getenv(data[1][j+1:k])
480	if var is not None:
481	data[1] = data[1][:i] + var + data[1][k+1:]
482	else:
483	my_error('Could not find variable '
484	+ data[1][j+1:k] + ' while reading ' +
485	self.controlfile)
486	setattr(self, data[0].lower() + '_expanded', data[1])
487	else:
488	if data[1].lower() != 'none':
489	setattr(self, data[0].lower(), data[1])
490	else:
491	setattr(self, data[0].lower(), None)
492	elif len(data) > 2:
493	setattr(self, data[0].lower(), (data[1:]))
494	else:
495	pass
496
497	return
498
499	def __str__(self):
500	'''Prepares a string which have all the ControlFile class attributes
501	with its associated values. Each attribute is printed in one line and
502	in alphabetical order.
503
504	Example
505	-------
506	'age': 10
507	'color': 'Spotted'
508	'kids': 0
509	'legs': 2
510	'name': 'Dog'
511	'smell': 'Alot'
512
513	Parameters
514	----------
515
516	Return
517	------
518	string
519	Single string of concatenated ControlFile class attributes
520	with their values
521	'''
522	import collections
523
524	attrs = vars(self).copy()
525	attrs = collections.OrderedDict(sorted(attrs.items()))
526
527	return '\n'.join("%s: %s" % item for item in attrs.items())
528
529	def assign_args_to_control(self, args):
530	'''Overwrites the existing ControlFile instance attributes with
531	the command line arguments.
532
533	Parameters
534	----------
535	args : Namespace
536	Contains the commandline arguments from script/program call.
537
538	Return
539	------
540
541	'''
542
543	# get dictionary of command line parameters and eliminate all
544	# parameters which are None (were not specified)
545	args_dict = vars(args)
546	arguments = {k : args_dict[k] for k in args_dict
547	if args_dict[k] != None}
548
549	# assign all passed command line arguments to ControlFile instance
550	for k, v in arguments.items():
551	setattr(self, str(k), v)
552
553	return
554
555	def assign_envs_to_control(self, envs):
556	'''Assigns the ECMWF environment parameter.
557
558	Parameters
559	----------
560	envs : dict of str
561	Contains the ECMWF environment parameternames "ECUID", "ECGID",
562	"DESTINATION" and "GATEWAY" with its corresponding values.
563	They were read from the file "ECMWF_ENV".
564
565	Return
566	------
567
568	'''
569
570	for k, v in envs.items():
571	setattr(self, str(k).lower(), str(v))
572
573	return
574
575	def check_conditions(self, queue):
576	'''Checks a couple of necessary attributes and conditions,
577	such as if they exist and contain values.
578	Otherwise set default values.
579
580	Parameters
581	----------
582	queue : str
583	Name of the queue if submitted to the ECMWF servers.
584	Used to check if ecuid, ecgid, gateway and destination
585	are set correctly and are not empty.
586
587	Return
588	------
589
590	'''
591	check_logicals_type(self, self.logicals)
592
593	self.mailfail = check_mail(self.mailfail)
594
595	self.mailops = check_mail(self.mailops)
596
597	check_queue(queue, self.gateway, self.destination,
598	self.ecuid, self.ecgid)
599
600	self.outputdir, self.installdir = check_pathes(self.inputdir,
601	self.outputdir,
602	self.installdir,
603	self.flexextractdir)
604
605	self.start_date, self.end_date = check_dates(self.start_date,
606	self.end_date)
607
608	self.basetime = check_basetime(self.basetime)
609
610	self.levelist, self.level = check_levels(self.levelist, self.level)
611
612	self.step = check_step(self.step)
613
614	self.maxstep = check_maxstep(self.maxstep, self.step)
615
616	check_request(self.request,
617	os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
618
619	check_public(self.public, self.dataset, self.marsclass)
620
621	self.type = check_type(self.type, self.step)
622
623	self.time = check_time(self.time)
624
625	self.purefc = check_purefc(self.type)
626
627	self.type, self.time, self.step = check_len_type_time_step(self.type,
628	self.time,
629	self.step,
630	self.maxstep,
631	self.purefc)
632
633	self.acctype = check_acctype(self.acctype, self.type)
634
635	self.acctime = check_acctime(self.acctime, self.marsclass,
636	self.purefc, self.time)
637
638	self.accmaxstep = check_accmaxstep(self.accmaxstep, self.marsclass,
639	self.purefc, self.maxstep)
640
641	self.grid = check_grid(self.grid)
642
643	self.area = check_area(self.grid, self.area, self.upper, self.lower,
644	self.left, self.right)
645
646	self.addpar = check_addpar(self.addpar)
647
648	self.job_chunk = check_job_chunk(self.job_chunk)
649
650	self.number = check_number(self.number)
651
652	return
653
654	def to_list(self):
655	'''Just generates a list of strings containing the attributes and
656	assigned values except the attributes "_expanded", "exedir",
657	"flexextractdir" and "installdir".
658
659	Parameters
660	----------
661
662	Return
663	------
664	l : list of *
665	A sorted list of the all ControlFile class attributes with
666	their values except the attributes "_expanded", "exedir",
667	"flexextractdir" and "installdir".
668	'''
669
670	import collections
671
672	attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
673
674	l = list()
675
676	for item in attrs.items():
677	if '_expanded' in item[0]:
678	pass
679	elif 'exedir' in item[0]:
680	pass
681	elif 'installdir' in item[0]:
682	pass
683	elif 'flexextractdir' in item[0]:
684	pass
685	else:
686	if isinstance(item[1], list):
687	stot = ''
688	for s in item[1]:
689	stot += s + ' '
690
691	l.append("%s %s\n" % (item[0], stot))
692	else:
693	l.append("%s %s\n" % item)
694
695	return sorted(l)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: