Context Navigation

source: flex_extract.git/Source/Python/Classes/ControlFile.py @ a676cf7

dev

Last change on this file since a676cf7 was a676cf7, checked in by Anne Tipka <anne.tipka@…>, 18 months ago
modifications and adjustments to dates and times for correctly retrieving analysis and forecast data in operational mode
Property mode set to `100644`
File size: 23.6 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - applied some minor modifications in programming style/structure
14	# - changed name of class Control to ControlFile for more
15	# self-explanation naming
16	# - outsource of class ControlFile
17	# - initialisation of class attributes ( to avoid high number of
18	# conditional statements and set default values )
19	# - divided assignment of attributes and the check of conditions
20	# - outsourced the commandline argument assignments to control attributes
21	# June 2020 - Anne Philipp
22	# - update default makefile to None
23	# August 2020 - Leopold Haimberger
24	# - added a class parameter for system installation path
25	# October 2022 - Anne Tipka (previously Philipp)
26	# - added class parameter for packingType (compression) of grib messages
27	# - added class parameter for event trigger for time critical jobs
28	#
29	# @License:
30	# (C) Copyright 2014-2020.
31	# Anne Philipp, Leopold Haimberger
32	#
33	# SPDX-License-Identifier: CC-BY-4.0
34	#
35	# This work is licensed under the Creative Commons Attribution 4.0
36	# International License. To view a copy of this license, visit
37	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
38	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
39	#*******************************************************************************
40
41	# ------------------------------------------------------------------------------
42	# MODULES
43	# ------------------------------------------------------------------------------
44	from __future__ import print_function
45
46	import os
47	import sys
48
49	# software specific classes and modules from flex_extract
50	#pylint: disable=wrong-import-position
51	sys.path.append('../')
52	import _config
53	from Mods.tools import my_error
54	from Mods.checks import (check_grid, check_area, check_levels, check_purefc,
55	check_step, check_mail, check_queue, check_pathes,
56	check_dates, check_maxstep, check_type, check_request,
57	check_basetime, check_public, check_acctype,
58	check_acctime, check_accmaxstep, check_time,
59	check_logicals_type, check_len_type_time_step,
60	check_addpar, check_job_chunk, check_number)
61	#pylint: enable=wrong-import-position
62
63	# ------------------------------------------------------------------------------
64	# CLASS
65	# ------------------------------------------------------------------------------
66	class ControlFile(object):
67	'''
68	Contains the information which are stored in the CONTROL files.
69
70	The CONTROL file is the steering part of the FLEXPART extraction
71	software. All necessary parameters needed to retrieve the data fields
72	from the MARS archive for driving FLEXPART are set in a CONTROL file.
73	Some specific parameters like the start and end dates can be overwritten
74	by the command line parameters, but in generall all parameters needed
75	for a complete set of fields for FLEXPART can be set in the CONTROL file.
76
77	Attributes
78	----------
79	controlfile : str
80	The name of the control file to be processed. Default value is the
81	filename passed to the init function when initialised.
82
83	start_date : str
84	The first day of the retrieval period. Default value is None.
85
86	end_date :str
87	The last day of the retrieval period. Default value is None.
88
89	date_chunk : int
90	Length of period for a single mars retrieval. Default value is 3.
91
92	dtime :str
93	The time step in hours. Default value is None.
94
95	basetime : int
96	The time for a half day retrieval. The 12 hours upfront are to be
97	retrieved. Default value is None.
98
99	maxstep : int
100	The maximum forecast step for non flux data. Default value is None.
101
102	type : list of str
103	List of field type per retrieving hour. Default value is None.
104
105	time : list of str
106	List of retrieving times in hours. Default valuer is None.
107
108	step : list of str or str
109	List of forecast time steps in hours for non flux data.
110	Default value is None.
111
112	acctype : str
113	The field type for the accumulated forecast fields.
114	Default value is None.
115
116	acctime : str
117	The starting time of the accumulated forecasts. Default value is None.
118
119	accmaxstep : int
120	The maximum forecast step for the accumulated forecast fields
121	(flux data). Default value is None.
122
123	marsclass : str
124	Characterisation of dataset. Default value is None.
125
126	dataset : str
127	For public datasets there is the specific naming and parameter
128	dataset which has to be used to characterize the type of
129	data. Default value is None.
130
131	stream : str
132	Identifies the forecasting system used to generate the data.
133	Default value is None.
134
135	number : str
136	Selects the member in ensemble forecast run. Default value is 'OFF'.
137
138	expver : str
139	The version number of the dataset. Default value is '1'.
140
141	gaussian : str
142	This parameter is deprecated and should no longer be used.
143	Specifies the desired type of Gaussian grid for the output.
144	Default value is an empty string ''.
145
146	grid : str
147	Specifies the output grid which can be either a Gaussian grid
148	or a Latitude/Longitude grid. Default value is None.
149
150	area : str
151	Specifies the desired sub-area of data to be extracted.
152	Default value is None.
153
154	left : str
155	The western most longitude of the area to be extracted.
156	Default value is None.
157
158	lower : str
159	The southern most latitude of the area to be extracted.
160	Default value is None.
161
162	upper : str
163	The northern most latitued of the area to be extracted.
164	Default value is None.
165
166	right : str
167	The eastern most longitude of the area to be extracted.
168	Default value is None.
169
170	level : str
171	Specifies the maximum level. Default value is None.
172
173	levelist : str
174	Specifies the required level list. Default value is None.
175
176	resol : str
177	Specifies the desired triangular truncation of retrieved data,
178	before carrying out any other selected post-processing.
179	Default value is None.
180
181	gauss : int
182	Switch to select gaussian fields (1) or regular lat/lon (0).
183	Default value is 0.
184
185	accuracy : int
186	Specifies the number of bits per value to be used in the
187	generated GRIB coded fields. Default value is 24.
188
189	omega : int
190	Switch to select omega retrieval (1) or not (0). Default value is 0.
191
192	omegadiff : int
193	Switch to decide to calculate Omega and Dps/Dt from continuity
194	equation for diagnostic purposes (1) or not (0). Default value is 0.
195
196	eta : int
197	Switch to select direct retrieval of etadot from MARS (1) or
198	wether it has to be calculated (0). Then Default value is 0.
199
200	etadiff : int
201	Switch to select calculation of etadot and Dps/Dt from continuity
202	equation for diagnostic purposes (1) or not (0). Default value is 0.
203
204	etapar : int
205	GRIB parameter Id for etadot fields. Default value is 77.
206
207	dpdeta : int
208	Switch to select multiplication of etadot with dpdeta.
209	Default value is 1.
210
211	smooth : int
212	Spectral truncation of ETADOT after calculation on Gaussian grid.
213	Default value is 0.
214
215	format : str
216	The format of the GRIB data. Default value is 'GRIB1'.
217
218	addpar : str
219	List of additional surface level ECMWF parameter to be retrieved.
220	Default value is None.
221
222	prefix : str
223	Prefix string for the final FLEXPART/FLEXTRA ready input files.
224	Default value is 'EN'.
225
226	cwc : int
227	Switch to select wether the sum of cloud liquid water content and
228	cloud ice water content should be retrieved. Default value is 0.
229
230	wrf : int
231	Switch to select further parameters for retrievment to support
232	WRF simulations. Default value is 0.
233
234	compression : str
235	Compression type for grib messages. Standard is "grid_simple".
236
237	ecfsdir : str
238	Path to the ECMWF storage 'ectmp:/${USER}/econdemand/'
239
240	mailfail : list of str
241	Email list for sending error log files from ECMWF servers.
242	The email addresses should be seperated by a comma.
243	Default value is ['${USER}'].
244
245	mailops : list of str
246	Email list for sending operational log files from ECMWF servers.
247	The email addresses should be seperated by a comma.
248	Default value is ['${USER}'].
249
250	ecstorage : int
251	Switch to select storage of FLEXPART ready output files
252	in the ECFS file system. Default value is 0.
253
254	ectrans : int
255	Switch to select the transfer of FLEXPART ready output files
256	to the gateway server. Default value is 0.
257
258	inputdir : str
259	Path to the temporary directory for the retrieval grib files and
260	other processing files. Default value is _config.PATH_INPUT_DIR.
261
262	outputdir : str
263	Path to the final directory where the final FLEXPART ready input
264	files are stored. Default value is None.
265
266	flexextractdir : str
267	Path to the flex_extract root directory. Default value is
268	_config.PATH_FLEXEXTRACT_DIR.
269
270	exedir : str
271	Path to the FORTRAN executable file. Default value is
272	_config.PATH_FORTRAN_SRC.
273
274	installdir : str
275	Path to a FLEXPART root directory. Default value is None.
276
277	makefile : str
278	Name of the makefile to be used for the Fortran program.
279	Default value is None.
280
281	destination : str
282	The remote destination which is used to transfer files
283	from ECMWF server to local gateway server. Default value is None.
284
285	gateway : str
286	The gateway server the user is using. Default value is None.
287
288	ecuid : str
289	The user id on ECMWF server. Default value is None.
290
291	ecgid : str
292	The group id on ECMWF server. Default value is None.
293
294	install_target : str
295	Defines the location where the installation is to be done.
296	Default value is None.
297
298	debug : int
299	Switch to keep temporary files at the end of postprocessing (1) or
300	to delete all temporary files except the final output files (0).
301	Default value is 0.
302
303	oper : int
304	Switch to prepare the operational job script. Start date, end date and
305	basetime will be prepared with environment variables.
306	Default value is 0.
307
308	request : int
309	Switch to select between just retrieving the data (0), writing the mars
310	parameter values to a csv file (1) or doing both (2).
311	Default value is 0.
312
313	public : int
314	Switch to select kind of ECMWF Web Api access and the
315	possible data sets. Public data sets (1) and Memberstate data sets (0).
316	Default value is 0.
317
318	ec_api : boolean
319	Tells wether the ECMWF Web API was able to load or not.
320	Default value is None.
321
322	cds_api : boolean
323	Tells wether the CDS API was able to load or not.
324	Default value is None.
325
326	purefc : int
327	Switch to decide wether the job is a pure forecast retrieval or
328	coupled with analysis data. Default value is 0.
329
330	rrint : int
331	Switch to select between old precipitation disaggregation method (0)
332	or the new IA3 disaggegration method (1). Default value is 0.
333
334	doubleelda : int
335	Switch to select the calculation of extra ensemble members for the
336	ELDA stream. It doubles the amount of retrieved ensemble members.
337
338	eventid : int
339	Id for one of the events for time critical jobs 1 at ECMWF servers.
340
341	eventjobname : str
342	The name the ecaccess-job-submit command will be given.
343	This is how the job will be recognized in the list of jobs.
344
345	logicals : list of str
346	List of the names of logical switches which controls the flow
347	of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
348	'etadiff', 'dpdeta', 'cwc', 'wrf', 'ecstorage',
349	'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint', 'doubleelda']
350	'''
351
352	def __init__(self, filename):
353	'''Initialises the instance of ControlFile class and defines
354	all class attributes with default values. Afterwards calls
355	function __read_controlfile__ to read parameter from Control file.
356
357	Parameters
358	----------
359	filename : str
360	Name of CONTROL file.
361
362	Return
363	------
364
365	'''
366
367	# list of all possible class attributes and their default values
368	self.controlfile = filename
369	self.start_date = None
370	self.end_date = None
371	self.date_chunk = 3
372	self.job_chunk = None
373	self.dtime = None
374	self.basetime = None
375	self.maxstep = None
376	self.type = None
377	self.time = None
378	self.step = None
379	self.acctype = None
380	self.acctime = None
381	self.accmaxstep = None
382	self.marsclass = None
383	self.dataset = None
384	self.stream = None
385	self.number = 'OFF'
386	self.expver = '1'
387	self.gaussian = ''
388	self.grid = None
389	self.area = ''
390	self.left = None
391	self.lower = None
392	self.upper = None
393	self.right = None
394	self.level = None
395	self.levelist = None
396	self.resol = None
397	self.gauss = 0
398	self.accuracy = 24
399	self.omega = 0
400	self.omegadiff = 0
401	self.eta = 0
402	self.etadiff = 0
403	self.etapar = 77
404	self.dpdeta = 1
405	self.smooth = 0
406	self.format = 'GRIB1'
407	self.addpar = None
408	self.prefix = 'EN'
409	self.cwc = 0
410	self.wrf = 0
411	self.compression = 'grid_simple'
412	self.ecfsdir = 'ectmp:/${USER}/econdemand/'
413	self.mailfail = ['${USER}']
414	self.mailops = ['${USER}']
415	self.ecstorage = 0
416	self.ectrans = 0
417	self.inputdir = _config.PATH_INPUT_DIR
418	self.outputdir = None
419	self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR
420	self.exedir = _config.PATH_FORTRAN_SRC
421	self.installdir = None
422	self.sysinstalldir = None
423	self.makefile = None
424	self.destination = None
425	self.gateway = None
426	self.ecuid = None
427	self.ecgid = None
428	self.install_target = None
429	self.debug = 0
430	self.oper = 0
431	self.request = 0
432	self.public = 0
433	self.ec_api = None
434	self.cds_api = None
435	self.purefc = 0
436	self.rrint = 0
437	self.doubleelda = 0
438	self.eventid = None
439	self.eventjobname = ''
440
441	self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
442	'dpdeta', 'cwc', 'wrf', 'ecstorage',
443	'ectrans', 'debug', 'oper', 'request', 'public',
444	'purefc', 'rrint', 'doubleelda']
445
446	self._read_controlfile()
447
448	return
449
450	def _read_controlfile(self):
451	'''Read CONTROL file and assign all CONTROL file variables.
452
453	Parameters
454	----------
455
456	Return
457	------
458
459	'''
460
461	try:
462	cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
463	with open(cfile) as f:
464	fdata = f.read().split('\n')
465	except IOError:
466	print('Could not read CONTROL file "' + cfile + '"')
467	print('Either it does not exist or its syntax is wrong.')
468	print('Try "' + sys.argv[0].split('/')[-1] + \
469	' -h" to print usage information')
470	sys.exit(1)
471
472	# go through every line and store parameter
473	for ldata in fdata:
474	if ldata and ldata[0] == '#':
475	# ignore comment line in control file
476	continue
477	if '#' in ldata:
478	# cut off comment
479	ldata = ldata.split('#')[0]
480	data = ldata.split()
481	if len(data) > 1:
482	if 'm_' in data[0].lower():
483	data[0] = data[0][2:]
484	if data[0].lower() == 'class':
485	data[0] = 'marsclass'
486	if data[0].lower() == 'day1':
487	data[0] = 'start_date'
488	if data[0].lower() == 'day2':
489	data[0] = 'end_date'
490	if len(data) == 2:
491	if '$' in data[1]:
492	setattr(self, data[0].lower(), data[1])
493	while '$' in data[1]:
494	i = data[1].index('$')
495	j = data[1].find('{')
496	k = data[1].find('}')
497	var = os.getenv(data[1][j+1:k])
498	if var is not None:
499	data[1] = data[1][:i] + var + data[1][k+1:]
500	else:
501	my_error('Could not find variable '
502	+ data[1][j+1:k] + ' while reading ' +
503	self.controlfile)
504	setattr(self, data[0].lower() + '_expanded', data[1])
505	else:
506	if data[1].lower() != 'none':
507	setattr(self, data[0].lower(), data[1])
508	else:
509	setattr(self, data[0].lower(), None)
510	elif len(data) > 2:
511	setattr(self, data[0].lower(), (data[1:]))
512	else:
513	pass
514
515	return
516
517	def __str__(self):
518	'''Prepares a string which have all the ControlFile class attributes
519	with its associated values. Each attribute is printed in one line and
520	in alphabetical order.
521
522	Example
523	-------
524	'age': 10
525	'color': 'Spotted'
526	'kids': 0
527	'legs': 2
528	'name': 'Dog'
529	'smell': 'Alot'
530
531	Parameters
532	----------
533
534	Return
535	------
536	string
537	Single string of concatenated ControlFile class attributes
538	with their values
539	'''
540	import collections
541
542	attrs = vars(self).copy()
543	attrs = collections.OrderedDict(sorted(attrs.items()))
544
545	return '\n'.join("%s: %s" % item for item in attrs.items())
546
547	def assign_args_to_control(self, args):
548	'''Overwrites the existing ControlFile instance attributes with
549	the command line arguments.
550
551	Parameters
552	----------
553	args : Namespace
554	Contains the commandline arguments from script/program call.
555
556	Return
557	------
558
559	'''
560
561	# get dictionary of command line parameters and eliminate all
562	# parameters which are None (were not specified)
563	args_dict = vars(args)
564	arguments = {k : args_dict[k] for k in args_dict
565	if args_dict[k] != None}
566
567	# assign all passed command line arguments to ControlFile instance
568	for k, v in arguments.items():
569	setattr(self, str(k), v)
570
571	return
572
573	def assign_envs_to_control(self, envs):
574	'''Assigns the ECMWF environment parameter.
575
576	Parameters
577	----------
578	envs : dict of str
579	Contains the ECMWF environment parameternames "ECUID", "ECGID",
580	"DESTINATION" and "GATEWAY" with its corresponding values.
581	They were read from the file "ECMWF_ENV".
582
583	Return
584	------
585
586	'''
587
588	for k, v in envs.items():
589	setattr(self, str(k).lower(), str(v))
590
591	return
592
593	def check_conditions(self, queue):
594	'''Checks a couple of necessary attributes and conditions,
595	such as if they exist and contain values.
596	Otherwise set default values.
597
598	Parameters
599	----------
600	queue : str
601	Name of the queue if submitted to the ECMWF servers.
602	Used to check if ecuid, ecgid, gateway and destination
603	are set correctly and are not empty.
604
605	Return
606	------
607
608	'''
609	check_logicals_type(self, self.logicals)
610
611	self.mailfail = check_mail(self.mailfail)
612
613	self.mailops = check_mail(self.mailops)
614
615	check_queue(queue, self.gateway, self.destination,
616	self.ecuid, self.ecgid)
617
618	self.outputdir, self.installdir = check_pathes(self.inputdir,
619	self.outputdir,
620	self.installdir,
621	self.flexextractdir)
622
623	self.start_date, self.end_date = check_dates(self.start_date,
624	self.end_date)
625
626	self.levelist, self.level = check_levels(self.levelist, self.level)
627
628	self.step = check_step(self.step)
629
630	self.maxstep = check_maxstep(self.maxstep, self.step)
631
632	check_request(self.request,
633	os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
634
635	check_public(self.public, self.dataset, self.marsclass)
636
637	self.type = check_type(self.type, self.step)
638
639	self.purefc = check_purefc(self.type)
640
641	self.basetime, self.time = check_basetime(self.basetime, self.time, self.purefc)
642
643	self.time = check_time(self.time)
644
645	self.type, self.time, self.step = check_len_type_time_step(self.type,
646	self.time,
647	self.step,
648	self.maxstep,
649	self.purefc)
650
651	self.acctype = check_acctype(self.acctype, self.type)
652
653	self.acctime = check_acctime(self.acctime, self.marsclass,
654	self.purefc, self.time,
655	self.basetime, self.oper)
656
657	self.accmaxstep = check_accmaxstep(self.accmaxstep, self.marsclass,
658	self.purefc, self.maxstep)
659
660	self.grid = check_grid(self.grid)
661
662	self.area = check_area(self.grid, self.area, self.upper, self.lower,
663	self.left, self.right)
664
665	self.addpar = check_addpar(self.addpar)
666
667	self.job_chunk = check_job_chunk(self.job_chunk)
668
669	self.number = check_number(self.number)
670
671	return
672
673	def to_list(self):
674	'''Just generates a list of strings containing the attributes and
675	assigned values except the attributes "_expanded", "exedir",
676	"flexextractdir" and "installdir".
677
678	Parameters
679	----------
680
681	Return
682	------
683	l : list of *
684	A sorted list of the all ControlFile class attributes with
685	their values except the attributes "_expanded", "exedir",
686	"flexextractdir" and "installdir".
687	'''
688
689	import collections
690
691	attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
692
693	l = list()
694
695	for item in attrs.items():
696	if '_expanded' in item[0]:
697	pass
698	elif 'exedir' in item[0]:
699	pass
700	elif 'installdir' in item[0]:
701	pass
702	elif 'flexextractdir' in item[0]:
703	pass
704	else:
705	if isinstance(item[1], list):
706	stot = ''
707	for s in item[1]:
708	stot += s + ' '
709
710	l.append("%s %s\n" % (item[0], stot))
711	else:
712	l.append("%s %s\n" % item)
713
714	return sorted(l)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: