Context Navigation

source: flex_extract.git/Source/Python/Classes/ControlFile.py @ 44174de

ctbtodev

Last change on this file since 44174de was 44174de, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
commented out the WRF parts since they are still under construction and added License SPDX tags
Property mode set to `100644`
File size: 22.9 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - applied some minor modifications in programming style/structure
14	# - changed name of class Control to ControlFile for more
15	# self-explanation naming
16	# - outsource of class ControlFile
17	# - initialisation of class attributes ( to avoid high number of
18	# conditional statements and set default values )
19	# - divided assignment of attributes and the check of conditions
20	# - outsourced the commandline argument assignments to control attributes
21	#
22	# @License:
23	# (C) Copyright 2014-2019.
24	# Anne Philipp, Leopold Haimberger
25	#
26	# SPDX-License-Identifier: CC-BY-4.0
27	#
28	# This work is licensed under the Creative Commons Attribution 4.0
29	# International License. To view a copy of this license, visit
30	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
31	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
32	#*******************************************************************************
33
34	# ------------------------------------------------------------------------------
35	# MODULES
36	# ------------------------------------------------------------------------------
37	from __future__ import print_function
38
39	import os
40	import re
41	import sys
42	import inspect
43
44	# software specific classes and modules from flex_extract
45	sys.path.append('../')
46	import _config
47	from Mods.tools import my_error, silent_remove
48	from Mods.checks import (check_grid, check_area, check_levels, check_purefc,
49	check_step, check_mail, check_queue, check_pathes,
50	check_dates, check_maxstep, check_type, check_request,
51	check_basetime, check_public, check_acctype,
52	check_acctime, check_accmaxstep, check_time,
53	check_logicals_type, check_len_type_time_step,
54	check_addpar, check_job_chunk, check_number)
55
56	# ------------------------------------------------------------------------------
57	# CLASS
58	# ------------------------------------------------------------------------------
59	class ControlFile(object):
60	'''
61	Contains the information which are stored in the CONTROL files.
62
63	The CONTROL file is the steering part of the FLEXPART extraction
64	software. All necessary parameters needed to retrieve the data fields
65	from the MARS archive for driving FLEXPART are set in a CONTROL file.
66	Some specific parameters like the start and end dates can be overwritten
67	by the command line parameters, but in generel all parameters needed
68	for a complete set of fields for FLEXPART can be set in the CONTROL file.
69
70	Attributes
71	----------
72	controlfile : str
73	The name of the control file to be processed. Default value is the
74	filename passed to the init function when initialised.
75
76	start_date : str
77	The first day of the retrieval period. Default value is None.
78
79	end_date :str
80	The last day of the retrieval period. Default value is None.
81
82	date_chunk : int
83	Length of period for a single mars retrieval. Default value is 3.
84
85	dtime :str
86	The time step in hours. Default value is None.
87
88	basetime : int
89	The time for a half day retrieval. The 12 hours upfront are to be
90	retrieved. Default value is None.
91
92	maxstep : int
93	The maximum forecast step for non flux data. Default value is None.
94
95	type : list of str
96	List of field type per retrieving hour. Default value is None.
97
98	time : list of str
99	List of retrieving times in hours. Default valuer is None.
100
101	step : list of str or str
102	List of forecast time steps in hours for non flux data.
103	Default value is None.
104
105	acctype : str
106	The field type for the accumulated forecast fields.
107	Default value is None.
108
109	acctime : str
110	The starting time of the accumulated forecasts. Default value is None.
111
112	accmaxstep : int
113	The maximum forecast step for the accumulated forecast fields
114	(flux data). Default value is None.
115
116	marsclass : str
117	Characterisation of dataset. Default value is None.
118
119	dataset : str
120	For public datasets there is the specific naming and parameter
121	dataset which has to be used to characterize the type of
122	data. Default value is None.
123
124	stream : str
125	Identifies the forecasting system used to generate the data.
126	Default value is None.
127
128	number : str
129	Selects the member in ensemble forecast run. Default value is 'OFF'.
130
131	expver : str
132	The version number of the dataset. Default value is '1'.
133
134	gaussian : str
135	This parameter is deprecated and should no longer be used.
136	Specifies the desired type of Gaussian grid for the output.
137	Default value is an empty string ''.
138
139	grid : str
140	Specifies the output grid which can be either a Gaussian grid
141	or a Latitude/Longitude grid. Default value is None.
142
143	area : str
144	Specifies the desired sub-area of data to be extracted.
145	Default value is None.
146
147	left : str
148	The western most longitude of the area to be extracted.
149	Default value is None.
150
151	lower : str
152	The southern most latitude of the area to be extracted.
153	Default value is None.
154
155	upper : str
156	The northern most latitued of the area to be extracted.
157	Default value is None.
158
159	right : str
160	The eastern most longitude of the area to be extracted.
161	Default value is None.
162
163	level : str
164	Specifies the maximum level. Default value is None.
165
166	levelist : str
167	Specifies the required level list. Default value is None.
168
169	resol : str
170	Specifies the desired triangular truncation of retrieved data,
171	before carrying out any other selected post-processing.
172	Default value is None.
173
174	gauss : int
175	Switch to select gaussian fields (1) or regular lat/lon (0).
176	Default value is 0.
177
178	accuracy : int
179	Specifies the number of bits per value to be used in the
180	generated GRIB coded fields. Default value is 24.
181
182	omega : int
183	Switch to select omega retrieval (1) or not (0). Default value is 0.
184
185	omegadiff : int
186	Switch to decide to calculate Omega and Dps/Dt from continuity
187	equation for diagnostic purposes (1) or not (0). Default value is 0.
188
189	eta : int
190	Switch to select direct retrieval of etadot from MARS (1) or
191	wether it has to be calculated (0). Then Default value is 0.
192
193	etadiff : int
194	Switch to select calculation of etadot and Dps/Dt from continuity
195	equation for diagnostic purposes (1) or not (0). Default value is 0.
196
197	etapar : int
198	GRIB parameter Id for etadot fields. Default value is 77.
199
200	dpdeta : int
201	Switch to select multiplication of etadot with dpdeta.
202	Default value is 1.
203
204	smooth : int
205	Spectral truncation of ETADOT after calculation on Gaussian grid.
206	Default value is 0.
207
208	format : str
209	The format of the GRIB data. Default value is 'GRIB1'.
210
211	addpar : str
212	List of additional surface level ECMWF parameter to be retrieved.
213	Default value is None.
214
215	prefix : str
216	Prefix string for the final FLEXPART/FLEXTRA ready input files.
217	Default value is 'EN'.
218
219	cwc : int
220	Switch to select wether the sum of cloud liquid water content and
221	cloud ice water content should be retrieved. Default value is 0.
222
223	wrf : int
224	Switch to select further parameters for retrievment to support
225	WRF simulations. Default value is 0.
226
227	ecfsdir : str
228	Path to the ECMWF storage 'ectmp:/${USER}/econdemand/'
229
230	mailfail : list of str
231	Email list for sending error log files from ECMWF servers.
232	The email addresses should be seperated by a comma.
233	Default value is ['${USER}'].
234
235	mailops : list of str
236	Email list for sending operational log files from ECMWF servers.
237	The email addresses should be seperated by a comma.
238	Default value is ['${USER}'].
239
240	grib2flexpart : int 0
241	Switch to select generation of preprocessed FLEXPART files ".fp".
242	If it is selected, the program grib2flexpart will try
243	to convert the flex_extract output files into ".fp" format.
244
245	ecstorage : int
246	Switch to select storage of FLEXPART ready output files
247	in the ECFS file system. Default value is 0.
248
249	ectrans : int
250	Switch to select the transfer of FLEXPART ready output files
251	to the gateway server. Default value is 0.
252
253	inputdir : str
254	Path to the temporary directory for the retrieval grib files and
255	other processing files. Default value is _config.PATH_INPUT_DIR.
256
257	outputdir : str
258	Path to the final directory where the final FLEXPART ready input
259	files are stored. Default value is None.
260
261	flexextractdir : str
262	Path to the flex_extract root directory. Default value is
263	_config.PATH_FLEXEXTRACT_DIR.
264
265	exedir : str
266	Path to the FORTRAN executable file. Default value is
267	_config.PATH_FORTRAN_SRC.
268
269	installdir : str
270	Path to a FLEXPART root directory. Default value is None.
271
272	makefile : str
273	Name of the makefile to be used for the Fortran program.
274	Default value is 'Makefile.gfortran'.
275
276	destination : str
277	The remote destination which is used to transfer files
278	from ECMWF server to local gateway server. Default value is None.
279
280	gateway : str
281	The gateway server the user is using. Default value is None.
282
283	ecuid : str
284	The user id on ECMWF server. Default value is None.
285
286	ecgid : str
287	The group id on ECMWF server. Default value is None.
288
289	install_target : str
290	Defines the location where the installation is to be done.
291	Default value is None.
292
293	debug : int
294	Switch to keep temporary files at the end of postprocessing (1) or
295	to delete all temporary files except the final output files (0).
296	Default value is 0.
297
298	oper : int
299	Switch to prepare the operational job script. Start date, end date and
300	basetime will be prepared with environment variables.
301	Default value is 0.
302
303	request : int
304	Switch to select between just retrieving the data (0), writing the mars
305	parameter values to a csv file (1) or doing both (2).
306	Default value is 0.
307
308	public : int
309	Switch to select kind of ECMWF Web Api access and the
310	possible data sets. Public data sets (1) and Memberstate data sets (0).
311	Default value is 0.
312
313	ec_api : boolean
314	Tells wether the ECMWF Web API was able to load or not.
315	Default value is None.
316
317	cds_api : boolean
318	Tells wether the CDS API was able to load or not.
319	Default value is None.
320
321	purefc : int
322	Switch to decide wether the job is a pure forecast retrieval or
323	coupled with analysis data. Default value is 0.
324
325	rrint : int
326	Switch to select between old precipitation disaggregation method (0)
327	or the new IA3 disaggegration method (1). Default value is 0.
328
329	doubleelda : int
330	Switch to select the calculation of extra ensemble members for the
331	ELDA stream. It doubles the amount of retrieved ensemble members.
332
333	logicals : list of str
334	List of the names of logical switches which controls the flow
335	of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
336	'etadiff', 'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
337	'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint', 'doubleelda']
338	'''
339
340	def __init__(self, filename):
341	'''Initialises the instance of ControlFile class and defines
342	all class attributes with default values. Afterwards calls
343	function __read_controlfile__ to read parameter from Control file.
344
345	Parameters
346	----------
347	filename : str
348	Name of CONTROL file.
349
350	Return
351	------
352
353	'''
354
355	# list of all possible class attributes and their default values
356	self.controlfile = filename
357	self.start_date = None
358	self.end_date = None
359	self.date_chunk = 3
360	self.job_chunk = None
361	self.dtime = None
362	self.basetime = None
363	self.maxstep = None
364	self.type = None
365	self.time = None
366	self.step = None
367	self.acctype = None
368	self.acctime = None
369	self.accmaxstep = None
370	self.marsclass = None
371	self.dataset = None
372	self.stream = None
373	self.number = 'OFF'
374	self.expver = '1'
375	self.gaussian = ''
376	self.grid = None
377	self.area = ''
378	self.left = None
379	self.lower = None
380	self.upper = None
381	self.right = None
382	self.level = None
383	self.levelist = None
384	self.resol = None
385	self.gauss = 0
386	self.accuracy = 24
387	self.omega = 0
388	self.omegadiff = 0
389	self.eta = 0
390	self.etadiff = 0
391	self.etapar = 77
392	self.dpdeta = 1
393	self.smooth = 0
394	self.format = 'GRIB1'
395	self.addpar = None
396	self.prefix = 'EN'
397	self.cwc = 0
398	self.wrf = 0
399	self.ecfsdir = 'ectmp:/${USER}/econdemand/'
400	self.mailfail = ['${USER}']
401	self.mailops = ['${USER}']
402	self.grib2flexpart = 0
403	self.ecstorage = 0
404	self.ectrans = 0
405	self.inputdir = _config.PATH_INPUT_DIR
406	self.outputdir = None
407	self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR
408	self.exedir = _config.PATH_FORTRAN_SRC
409	self.installdir = None
410	self.makefile = 'Makefile.gfortran'
411	self.destination = None
412	self.gateway = None
413	self.ecuid = None
414	self.ecgid = None
415	self.install_target = None
416	self.debug = 0
417	self.oper = 0
418	self.request = 0
419	self.public = 0
420	self.ec_api = None
421	self.cds_api = None
422	self.purefc = 0
423	self.rrint = 0
424	self.doubleelda = 0
425
426	self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
427	'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
428	'ectrans', 'debug', 'oper', 'request', 'public',
429	'purefc', 'rrint', 'doubleelda']
430
431	self._read_controlfile()
432
433	return
434
435	def _read_controlfile(self):
436	'''Read CONTROL file and assign all CONTROL file variables.
437
438	Parameters
439	----------
440
441	Return
442	------
443
444	'''
445
446	try:
447	cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
448	with open(cfile) as f:
449	fdata = f.read().split('\n')
450	except IOError:
451	print('Could not read CONTROL file "' + cfile + '"')
452	print('Either it does not exist or its syntax is wrong.')
453	print('Try "' + sys.argv[0].split('/')[-1] + \
454	' -h" to print usage information')
455	sys.exit(1)
456
457	# go through every line and store parameter
458	for ldata in fdata:
459	if ldata and ldata[0] == '#':
460	# ignore comment line in control file
461	continue
462	if '#' in ldata:
463	# cut off comment
464	ldata = ldata.split('#')[0]
465	data = ldata.split()
466	if len(data) > 1:
467	if 'm_' in data[0].lower():
468	data[0] = data[0][2:]
469	if data[0].lower() == 'class':
470	data[0] = 'marsclass'
471	if data[0].lower() == 'day1':
472	data[0] = 'start_date'
473	if data[0].lower() == 'day2':
474	data[0] = 'end_date'
475	if len(data) == 2:
476	if '$' in data[1]:
477	setattr(self, data[0].lower(), data[1])
478	while '$' in data[1]:
479	i = data[1].index('$')
480	j = data[1].find('{')
481	k = data[1].find('}')
482	var = os.getenv(data[1][j+1:k])
483	if var is not None:
484	data[1] = data[1][:i] + var + data[1][k+1:]
485	else:
486	my_error('Could not find variable '
487	+ data[1][j+1:k] + ' while reading ' +
488	self.controlfile)
489	setattr(self, data[0].lower() + '_expanded', data[1])
490	else:
491	if data[1].lower() != 'none':
492	setattr(self, data[0].lower(), data[1])
493	else:
494	setattr(self, data[0].lower(), None)
495	elif len(data) > 2:
496	setattr(self, data[0].lower(), (data[1:]))
497	else:
498	pass
499
500	return
501
502	def __str__(self):
503	'''Prepares a string which have all the ControlFile class attributes
504	with its associated values. Each attribute is printed in one line and
505	in alphabetical order.
506
507	Example
508	-------
509	'age': 10
510	'color': 'Spotted'
511	'kids': 0
512	'legs': 2
513	'name': 'Dog'
514	'smell': 'Alot'
515
516	Parameters
517	----------
518
519	Return
520	------
521	string
522	Single string of concatenated ControlFile class attributes
523	with their values
524	'''
525	import collections
526
527	attrs = vars(self).copy()
528	attrs = collections.OrderedDict(sorted(attrs.items()))
529
530	return '\n'.join("%s: %s" % item for item in attrs.items())
531
532	def assign_args_to_control(self, args):
533	'''Overwrites the existing ControlFile instance attributes with
534	the command line arguments.
535
536	Parameters
537	----------
538	args : Namespace
539	Contains the commandline arguments from script/program call.
540
541	Return
542	------
543
544	'''
545
546	# get dictionary of command line parameters and eliminate all
547	# parameters which are None (were not specified)
548	args_dict = vars(args)
549	arguments = {k : args_dict[k] for k in args_dict
550	if args_dict[k] != None}
551
552	# assign all passed command line arguments to ControlFile instance
553	for k, v in arguments.items():
554	setattr(self, str(k), v)
555
556	return
557
558	def assign_envs_to_control(self, envs):
559	'''Assigns the ECMWF environment parameter.
560
561	Parameters
562	----------
563	envs : dict of str
564	Contains the ECMWF environment parameternames "ECUID", "ECGID",
565	"DESTINATION" and "GATEWAY" with its corresponding values.
566	They were read from the file "ECMWF_ENV".
567
568	Return
569	------
570
571	'''
572
573	for k, v in envs.items():
574	setattr(self, str(k).lower(), str(v))
575
576	return
577
578	def check_conditions(self, queue):
579	'''Checks a couple of necessary attributes and conditions,
580	such as if they exist and contain values.
581	Otherwise set default values.
582
583	Parameters
584	----------
585	queue : str
586	Name of the queue if submitted to the ECMWF servers.
587	Used to check if ecuid, ecgid, gateway and destination
588	are set correctly and are not empty.
589
590	Return
591	------
592
593	'''
594	check_logicals_type(self, self.logicals)
595
596	self.mailfail = check_mail(self.mailfail)
597
598	self.mailops = check_mail(self.mailops)
599
600	check_queue(queue, self.gateway, self.destination,
601	self.ecuid, self.ecgid)
602
603	self.outputdir, self.installdir = check_pathes(self.inputdir,
604	self.outputdir, self.installdir, self.flexextractdir)
605
606	self.start_date, self.end_date = check_dates(self.start_date,
607	self.end_date)
608
609	self.basetime = check_basetime(self.basetime)
610
611	self.levelist, self.level = check_levels(self.levelist, self.level)
612
613	self.step = check_step(self.step, self.mailfail)
614
615	self.maxstep = check_maxstep(self.maxstep, self.step)
616
617	check_request(self.request,
618	os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
619
620	check_public(self.public, self.dataset)
621
622	self.type = check_type(self.type, self.step)
623
624	self.time = check_time(self.time)
625
626	self.purefc = check_purefc(self.type)
627
628	self.type, self.time, self.step = check_len_type_time_step(self.type,
629	self.time,
630	self.step,
631	self.maxstep,
632	self.purefc)
633
634	self.acctype = check_acctype(self.acctype, self.type)
635
636	self.acctime = check_acctime(self.acctime, self.marsclass,
637	self.purefc, self.time)
638
639	self.accmaxstep = check_accmaxstep(self.accmaxstep, self.marsclass,
640	self.purefc, self.maxstep)
641
642	self.grid = check_grid(self.grid)
643
644	self.area = check_area(self.grid, self.area, self.upper, self.lower,
645	self.left, self.right)
646
647	self.addpar = check_addpar(self.addpar)
648
649	self.job_chunk = check_job_chunk(self.job_chunk)
650
651	self.number = check_number(self.number, self.mailfail)
652
653	return
654
655	def to_list(self):
656	'''Just generates a list of strings containing the attributes and
657	assigned values except the attributes "_expanded", "exedir",
658	"flexextractdir" and "installdir".
659
660	Parameters
661	----------
662
663	Return
664	------
665	l : list of *
666	A sorted list of the all ControlFile class attributes with
667	their values except the attributes "_expanded", "exedir",
668	"flexextractdir" and "installdir".
669	'''
670
671	import collections
672
673	attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
674
675	l = list()
676
677	for item in attrs.items():
678	if '_expanded' in item[0]:
679	pass
680	elif 'exedir' in item[0]:
681	pass
682	elif 'installdir' in item[0]:
683	pass
684	elif 'flexextractdir' in item[0]:
685	pass
686	else:
687	if isinstance(item[1], list):
688	stot = ''
689	for s in item[1]:
690	stot += s + ' '
691
692	l.append("%s %s\n" % (item[0], stot))
693	else:
694	l.append("%s %s\n" % item)
695
696	return sorted(l)
697

Note: See TracBrowser for help on using the repository browser.

Download in other formats: