Context Navigation

source: flex_extract.git/Source/Python/Classes/ControlFile.py @ b9ce98c

dev

Last change on this file since b9ce98c was b9ce98c, checked in by Anne Tipka <anne.tipka@…>, 18 months ago
added CONTROL file option for grib compression type / set packingType parameter in grib messages
Property mode set to `100644`
File size: 23.0 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - applied some minor modifications in programming style/structure
14	# - changed name of class Control to ControlFile for more
15	# self-explanation naming
16	# - outsource of class ControlFile
17	# - initialisation of class attributes ( to avoid high number of
18	# conditional statements and set default values )
19	# - divided assignment of attributes and the check of conditions
20	# - outsourced the commandline argument assignments to control attributes
21	# June 2020 - Anne Philipp
22	# - update default makefile to None
23	# August 2020 - Leopold Haimberger
24	# - added a class parameter for system installation path
25	#
26	# @License:
27	# (C) Copyright 2014-2020.
28	# Anne Philipp, Leopold Haimberger
29	#
30	# SPDX-License-Identifier: CC-BY-4.0
31	#
32	# This work is licensed under the Creative Commons Attribution 4.0
33	# International License. To view a copy of this license, visit
34	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
35	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
36	#*******************************************************************************
37
38	# ------------------------------------------------------------------------------
39	# MODULES
40	# ------------------------------------------------------------------------------
41	from __future__ import print_function
42
43	import os
44	import sys
45
46	# software specific classes and modules from flex_extract
47	#pylint: disable=wrong-import-position
48	sys.path.append('../')
49	import _config
50	from Mods.tools import my_error
51	from Mods.checks import (check_grid, check_area, check_levels, check_purefc,
52	check_step, check_mail, check_queue, check_pathes,
53	check_dates, check_maxstep, check_type, check_request,
54	check_basetime, check_public, check_acctype,
55	check_acctime, check_accmaxstep, check_time,
56	check_logicals_type, check_len_type_time_step,
57	check_addpar, check_job_chunk, check_number)
58	#pylint: enable=wrong-import-position
59
60	# ------------------------------------------------------------------------------
61	# CLASS
62	# ------------------------------------------------------------------------------
63	class ControlFile(object):
64	'''
65	Contains the information which are stored in the CONTROL files.
66
67	The CONTROL file is the steering part of the FLEXPART extraction
68	software. All necessary parameters needed to retrieve the data fields
69	from the MARS archive for driving FLEXPART are set in a CONTROL file.
70	Some specific parameters like the start and end dates can be overwritten
71	by the command line parameters, but in generall all parameters needed
72	for a complete set of fields for FLEXPART can be set in the CONTROL file.
73
74	Attributes
75	----------
76	controlfile : str
77	The name of the control file to be processed. Default value is the
78	filename passed to the init function when initialised.
79
80	start_date : str
81	The first day of the retrieval period. Default value is None.
82
83	end_date :str
84	The last day of the retrieval period. Default value is None.
85
86	date_chunk : int
87	Length of period for a single mars retrieval. Default value is 3.
88
89	dtime :str
90	The time step in hours. Default value is None.
91
92	basetime : int
93	The time for a half day retrieval. The 12 hours upfront are to be
94	retrieved. Default value is None.
95
96	maxstep : int
97	The maximum forecast step for non flux data. Default value is None.
98
99	type : list of str
100	List of field type per retrieving hour. Default value is None.
101
102	time : list of str
103	List of retrieving times in hours. Default valuer is None.
104
105	step : list of str or str
106	List of forecast time steps in hours for non flux data.
107	Default value is None.
108
109	acctype : str
110	The field type for the accumulated forecast fields.
111	Default value is None.
112
113	acctime : str
114	The starting time of the accumulated forecasts. Default value is None.
115
116	accmaxstep : int
117	The maximum forecast step for the accumulated forecast fields
118	(flux data). Default value is None.
119
120	marsclass : str
121	Characterisation of dataset. Default value is None.
122
123	dataset : str
124	For public datasets there is the specific naming and parameter
125	dataset which has to be used to characterize the type of
126	data. Default value is None.
127
128	stream : str
129	Identifies the forecasting system used to generate the data.
130	Default value is None.
131
132	number : str
133	Selects the member in ensemble forecast run. Default value is 'OFF'.
134
135	expver : str
136	The version number of the dataset. Default value is '1'.
137
138	gaussian : str
139	This parameter is deprecated and should no longer be used.
140	Specifies the desired type of Gaussian grid for the output.
141	Default value is an empty string ''.
142
143	grid : str
144	Specifies the output grid which can be either a Gaussian grid
145	or a Latitude/Longitude grid. Default value is None.
146
147	area : str
148	Specifies the desired sub-area of data to be extracted.
149	Default value is None.
150
151	left : str
152	The western most longitude of the area to be extracted.
153	Default value is None.
154
155	lower : str
156	The southern most latitude of the area to be extracted.
157	Default value is None.
158
159	upper : str
160	The northern most latitued of the area to be extracted.
161	Default value is None.
162
163	right : str
164	The eastern most longitude of the area to be extracted.
165	Default value is None.
166
167	level : str
168	Specifies the maximum level. Default value is None.
169
170	levelist : str
171	Specifies the required level list. Default value is None.
172
173	resol : str
174	Specifies the desired triangular truncation of retrieved data,
175	before carrying out any other selected post-processing.
176	Default value is None.
177
178	gauss : int
179	Switch to select gaussian fields (1) or regular lat/lon (0).
180	Default value is 0.
181
182	accuracy : int
183	Specifies the number of bits per value to be used in the
184	generated GRIB coded fields. Default value is 24.
185
186	omega : int
187	Switch to select omega retrieval (1) or not (0). Default value is 0.
188
189	omegadiff : int
190	Switch to decide to calculate Omega and Dps/Dt from continuity
191	equation for diagnostic purposes (1) or not (0). Default value is 0.
192
193	eta : int
194	Switch to select direct retrieval of etadot from MARS (1) or
195	wether it has to be calculated (0). Then Default value is 0.
196
197	etadiff : int
198	Switch to select calculation of etadot and Dps/Dt from continuity
199	equation for diagnostic purposes (1) or not (0). Default value is 0.
200
201	etapar : int
202	GRIB parameter Id for etadot fields. Default value is 77.
203
204	dpdeta : int
205	Switch to select multiplication of etadot with dpdeta.
206	Default value is 1.
207
208	smooth : int
209	Spectral truncation of ETADOT after calculation on Gaussian grid.
210	Default value is 0.
211
212	format : str
213	The format of the GRIB data. Default value is 'GRIB1'.
214
215	addpar : str
216	List of additional surface level ECMWF parameter to be retrieved.
217	Default value is None.
218
219	prefix : str
220	Prefix string for the final FLEXPART/FLEXTRA ready input files.
221	Default value is 'EN'.
222
223	cwc : int
224	Switch to select wether the sum of cloud liquid water content and
225	cloud ice water content should be retrieved. Default value is 0.
226
227	wrf : int
228	Switch to select further parameters for retrievment to support
229	WRF simulations. Default value is 0.
230
231	compression : str
232	Compression type for grib messages. Standard is "grid_simple".
233
234	ecfsdir : str
235	Path to the ECMWF storage 'ectmp:/${USER}/econdemand/'
236
237	mailfail : list of str
238	Email list for sending error log files from ECMWF servers.
239	The email addresses should be seperated by a comma.
240	Default value is ['${USER}'].
241
242	mailops : list of str
243	Email list for sending operational log files from ECMWF servers.
244	The email addresses should be seperated by a comma.
245	Default value is ['${USER}'].
246
247	ecstorage : int
248	Switch to select storage of FLEXPART ready output files
249	in the ECFS file system. Default value is 0.
250
251	ectrans : int
252	Switch to select the transfer of FLEXPART ready output files
253	to the gateway server. Default value is 0.
254
255	inputdir : str
256	Path to the temporary directory for the retrieval grib files and
257	other processing files. Default value is _config.PATH_INPUT_DIR.
258
259	outputdir : str
260	Path to the final directory where the final FLEXPART ready input
261	files are stored. Default value is None.
262
263	flexextractdir : str
264	Path to the flex_extract root directory. Default value is
265	_config.PATH_FLEXEXTRACT_DIR.
266
267	exedir : str
268	Path to the FORTRAN executable file. Default value is
269	_config.PATH_FORTRAN_SRC.
270
271	installdir : str
272	Path to a FLEXPART root directory. Default value is None.
273
274	makefile : str
275	Name of the makefile to be used for the Fortran program.
276	Default value is None.
277
278	destination : str
279	The remote destination which is used to transfer files
280	from ECMWF server to local gateway server. Default value is None.
281
282	gateway : str
283	The gateway server the user is using. Default value is None.
284
285	ecuid : str
286	The user id on ECMWF server. Default value is None.
287
288	ecgid : str
289	The group id on ECMWF server. Default value is None.
290
291	install_target : str
292	Defines the location where the installation is to be done.
293	Default value is None.
294
295	debug : int
296	Switch to keep temporary files at the end of postprocessing (1) or
297	to delete all temporary files except the final output files (0).
298	Default value is 0.
299
300	oper : int
301	Switch to prepare the operational job script. Start date, end date and
302	basetime will be prepared with environment variables.
303	Default value is 0.
304
305	request : int
306	Switch to select between just retrieving the data (0), writing the mars
307	parameter values to a csv file (1) or doing both (2).
308	Default value is 0.
309
310	public : int
311	Switch to select kind of ECMWF Web Api access and the
312	possible data sets. Public data sets (1) and Memberstate data sets (0).
313	Default value is 0.
314
315	ec_api : boolean
316	Tells wether the ECMWF Web API was able to load or not.
317	Default value is None.
318
319	cds_api : boolean
320	Tells wether the CDS API was able to load or not.
321	Default value is None.
322
323	purefc : int
324	Switch to decide wether the job is a pure forecast retrieval or
325	coupled with analysis data. Default value is 0.
326
327	rrint : int
328	Switch to select between old precipitation disaggregation method (0)
329	or the new IA3 disaggegration method (1). Default value is 0.
330
331	doubleelda : int
332	Switch to select the calculation of extra ensemble members for the
333	ELDA stream. It doubles the amount of retrieved ensemble members.
334
335	logicals : list of str
336	List of the names of logical switches which controls the flow
337	of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
338	'etadiff', 'dpdeta', 'cwc', 'wrf', 'ecstorage',
339	'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint', 'doubleelda']
340	'''
341
342	def __init__(self, filename):
343	'''Initialises the instance of ControlFile class and defines
344	all class attributes with default values. Afterwards calls
345	function __read_controlfile__ to read parameter from Control file.
346
347	Parameters
348	----------
349	filename : str
350	Name of CONTROL file.
351
352	Return
353	------
354
355	'''
356
357	# list of all possible class attributes and their default values
358	self.controlfile = filename
359	self.start_date = None
360	self.end_date = None
361	self.date_chunk = 3
362	self.job_chunk = None
363	self.dtime = None
364	self.basetime = None
365	self.maxstep = None
366	self.type = None
367	self.time = None
368	self.step = None
369	self.acctype = None
370	self.acctime = None
371	self.accmaxstep = None
372	self.marsclass = None
373	self.dataset = None
374	self.stream = None
375	self.number = 'OFF'
376	self.expver = '1'
377	self.gaussian = ''
378	self.grid = None
379	self.area = ''
380	self.left = None
381	self.lower = None
382	self.upper = None
383	self.right = None
384	self.level = None
385	self.levelist = None
386	self.resol = None
387	self.gauss = 0
388	self.accuracy = 24
389	self.omega = 0
390	self.omegadiff = 0
391	self.eta = 0
392	self.etadiff = 0
393	self.etapar = 77
394	self.dpdeta = 1
395	self.smooth = 0
396	self.format = 'GRIB1'
397	self.addpar = None
398	self.prefix = 'EN'
399	self.cwc = 0
400	self.wrf = 0
401	self.compression = 'grid_simple'
402	self.ecfsdir = 'ectmp:/${USER}/econdemand/'
403	self.mailfail = ['${USER}']
404	self.mailops = ['${USER}']
405	self.ecstorage = 0
406	self.ectrans = 0
407	self.inputdir = _config.PATH_INPUT_DIR
408	self.outputdir = None
409	self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR
410	self.exedir = _config.PATH_FORTRAN_SRC
411	self.installdir = None
412	self.sysinstalldir = None
413	self.makefile = None
414	self.destination = None
415	self.gateway = None
416	self.ecuid = None
417	self.ecgid = None
418	self.install_target = None
419	self.debug = 0
420	self.oper = 0
421	self.request = 0
422	self.public = 0
423	self.ec_api = None
424	self.cds_api = None
425	self.purefc = 0
426	self.rrint = 0
427	self.doubleelda = 0
428
429	self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
430	'dpdeta', 'cwc', 'wrf', 'ecstorage',
431	'ectrans', 'debug', 'oper', 'request', 'public',
432	'purefc', 'rrint', 'doubleelda']
433
434	self._read_controlfile()
435
436	return
437
438	def _read_controlfile(self):
439	'''Read CONTROL file and assign all CONTROL file variables.
440
441	Parameters
442	----------
443
444	Return
445	------
446
447	'''
448
449	try:
450	cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
451	with open(cfile) as f:
452	fdata = f.read().split('\n')
453	except IOError:
454	print('Could not read CONTROL file "' + cfile + '"')
455	print('Either it does not exist or its syntax is wrong.')
456	print('Try "' + sys.argv[0].split('/')[-1] + \
457	' -h" to print usage information')
458	sys.exit(1)
459
460	# go through every line and store parameter
461	for ldata in fdata:
462	if ldata and ldata[0] == '#':
463	# ignore comment line in control file
464	continue
465	if '#' in ldata:
466	# cut off comment
467	ldata = ldata.split('#')[0]
468	data = ldata.split()
469	if len(data) > 1:
470	if 'm_' in data[0].lower():
471	data[0] = data[0][2:]
472	if data[0].lower() == 'class':
473	data[0] = 'marsclass'
474	if data[0].lower() == 'day1':
475	data[0] = 'start_date'
476	if data[0].lower() == 'day2':
477	data[0] = 'end_date'
478	if len(data) == 2:
479	if '$' in data[1]:
480	setattr(self, data[0].lower(), data[1])
481	while '$' in data[1]:
482	i = data[1].index('$')
483	j = data[1].find('{')
484	k = data[1].find('}')
485	var = os.getenv(data[1][j+1:k])
486	if var is not None:
487	data[1] = data[1][:i] + var + data[1][k+1:]
488	else:
489	my_error('Could not find variable '
490	+ data[1][j+1:k] + ' while reading ' +
491	self.controlfile)
492	setattr(self, data[0].lower() + '_expanded', data[1])
493	else:
494	if data[1].lower() != 'none':
495	setattr(self, data[0].lower(), data[1])
496	else:
497	setattr(self, data[0].lower(), None)
498	elif len(data) > 2:
499	setattr(self, data[0].lower(), (data[1:]))
500	else:
501	pass
502
503	return
504
505	def __str__(self):
506	'''Prepares a string which have all the ControlFile class attributes
507	with its associated values. Each attribute is printed in one line and
508	in alphabetical order.
509
510	Example
511	-------
512	'age': 10
513	'color': 'Spotted'
514	'kids': 0
515	'legs': 2
516	'name': 'Dog'
517	'smell': 'Alot'
518
519	Parameters
520	----------
521
522	Return
523	------
524	string
525	Single string of concatenated ControlFile class attributes
526	with their values
527	'''
528	import collections
529
530	attrs = vars(self).copy()
531	attrs = collections.OrderedDict(sorted(attrs.items()))
532
533	return '\n'.join("%s: %s" % item for item in attrs.items())
534
535	def assign_args_to_control(self, args):
536	'''Overwrites the existing ControlFile instance attributes with
537	the command line arguments.
538
539	Parameters
540	----------
541	args : Namespace
542	Contains the commandline arguments from script/program call.
543
544	Return
545	------
546
547	'''
548
549	# get dictionary of command line parameters and eliminate all
550	# parameters which are None (were not specified)
551	args_dict = vars(args)
552	arguments = {k : args_dict[k] for k in args_dict
553	if args_dict[k] != None}
554
555	# assign all passed command line arguments to ControlFile instance
556	for k, v in arguments.items():
557	setattr(self, str(k), v)
558
559	return
560
561	def assign_envs_to_control(self, envs):
562	'''Assigns the ECMWF environment parameter.
563
564	Parameters
565	----------
566	envs : dict of str
567	Contains the ECMWF environment parameternames "ECUID", "ECGID",
568	"DESTINATION" and "GATEWAY" with its corresponding values.
569	They were read from the file "ECMWF_ENV".
570
571	Return
572	------
573
574	'''
575
576	for k, v in envs.items():
577	setattr(self, str(k).lower(), str(v))
578
579	return
580
581	def check_conditions(self, queue):
582	'''Checks a couple of necessary attributes and conditions,
583	such as if they exist and contain values.
584	Otherwise set default values.
585
586	Parameters
587	----------
588	queue : str
589	Name of the queue if submitted to the ECMWF servers.
590	Used to check if ecuid, ecgid, gateway and destination
591	are set correctly and are not empty.
592
593	Return
594	------
595
596	'''
597	check_logicals_type(self, self.logicals)
598
599	self.mailfail = check_mail(self.mailfail)
600
601	self.mailops = check_mail(self.mailops)
602
603	check_queue(queue, self.gateway, self.destination,
604	self.ecuid, self.ecgid)
605
606	self.outputdir, self.installdir = check_pathes(self.inputdir,
607	self.outputdir,
608	self.installdir,
609	self.flexextractdir)
610
611	self.start_date, self.end_date = check_dates(self.start_date,
612	self.end_date)
613
614	self.basetime = check_basetime(self.basetime)
615
616	self.levelist, self.level = check_levels(self.levelist, self.level)
617
618	self.step = check_step(self.step)
619
620	self.maxstep = check_maxstep(self.maxstep, self.step)
621
622	check_request(self.request,
623	os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
624
625	check_public(self.public, self.dataset, self.marsclass)
626
627	self.type = check_type(self.type, self.step)
628
629	self.time = check_time(self.time)
630
631	self.purefc = check_purefc(self.type)
632
633	self.type, self.time, self.step = check_len_type_time_step(self.type,
634	self.time,
635	self.step,
636	self.maxstep,
637	self.purefc)
638
639	self.acctype = check_acctype(self.acctype, self.type)
640
641	self.acctime = check_acctime(self.acctime, self.marsclass,
642	self.purefc, self.time)
643
644	self.accmaxstep = check_accmaxstep(self.accmaxstep, self.marsclass,
645	self.purefc, self.maxstep)
646
647	self.grid = check_grid(self.grid)
648
649	self.area = check_area(self.grid, self.area, self.upper, self.lower,
650	self.left, self.right)
651
652	self.addpar = check_addpar(self.addpar)
653
654	self.job_chunk = check_job_chunk(self.job_chunk)
655
656	self.number = check_number(self.number)
657
658	return
659
660	def to_list(self):
661	'''Just generates a list of strings containing the attributes and
662	assigned values except the attributes "_expanded", "exedir",
663	"flexextractdir" and "installdir".
664
665	Parameters
666	----------
667
668	Return
669	------
670	l : list of *
671	A sorted list of the all ControlFile class attributes with
672	their values except the attributes "_expanded", "exedir",
673	"flexextractdir" and "installdir".
674	'''
675
676	import collections
677
678	attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
679
680	l = list()
681
682	for item in attrs.items():
683	if '_expanded' in item[0]:
684	pass
685	elif 'exedir' in item[0]:
686	pass
687	elif 'installdir' in item[0]:
688	pass
689	elif 'flexextractdir' in item[0]:
690	pass
691	else:
692	if isinstance(item[1], list):
693	stot = ''
694	for s in item[1]:
695	stot += s + ' '
696
697	l.append("%s %s\n" % (item[0], stot))
698	else:
699	l.append("%s %s\n" % item)
700
701	return sorted(l)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: