Context Navigation

source: flex_extract.git/Source/Python/Classes/ControlFile.py @ 0a75335

ctbtodev

Last change on this file since 0a75335 was 0a75335, checked in by Anne Philipp <anne.philipp@…>, 4 years ago
changed cds api retrieval for single levels to retrieve from CS3 disks instead of MARS - severe performance boost
Property mode set to `100644`
File size: 22.8 KB

Line
1	#!/usr/bin/env python3
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - applied some minor modifications in programming style/structure
14	# - changed name of class Control to ControlFile for more
15	# self-explanation naming
16	# - outsource of class ControlFile
17	# - initialisation of class attributes ( to avoid high number of
18	# conditional statements and set default values )
19	# - divided assignment of attributes and the check of conditions
20	# - outsourced the commandline argument assignments to control attributes
21	# June 2020 - Anne Philipp
22	# - update default makefile to None
23	#
24	# @License:
25	# (C) Copyright 2014-2020.
26	# Anne Philipp, Leopold Haimberger
27	#
28	# SPDX-License-Identifier: CC-BY-4.0
29	#
30	# This work is licensed under the Creative Commons Attribution 4.0
31	# International License. To view a copy of this license, visit
32	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
33	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
34	#*******************************************************************************
35
36	# ------------------------------------------------------------------------------
37	# MODULES
38	# ------------------------------------------------------------------------------
39	from __future__ import print_function
40
41	import os
42	import sys
43
44	# software specific classes and modules from flex_extract
45	#pylint: disable=wrong-import-position
46	sys.path.append('../')
47	import _config
48	from Mods.tools import my_error
49	from Mods.checks import (check_grid, check_area, check_levels, check_purefc,
50	check_step, check_mail, check_queue, check_pathes,
51	check_dates, check_maxstep, check_type, check_request,
52	check_basetime, check_public, check_acctype,
53	check_acctime, check_accmaxstep, check_time,
54	check_logicals_type, check_len_type_time_step,
55	check_addpar, check_job_chunk, check_number)
56	#pylint: enable=wrong-import-position
57
58	# ------------------------------------------------------------------------------
59	# CLASS
60	# ------------------------------------------------------------------------------
61	class ControlFile(object):
62	'''
63	Contains the information which are stored in the CONTROL files.
64
65	The CONTROL file is the steering part of the FLEXPART extraction
66	software. All necessary parameters needed to retrieve the data fields
67	from the MARS archive for driving FLEXPART are set in a CONTROL file.
68	Some specific parameters like the start and end dates can be overwritten
69	by the command line parameters, but in generall all parameters needed
70	for a complete set of fields for FLEXPART can be set in the CONTROL file.
71
72	Attributes
73	----------
74	controlfile : str
75	The name of the control file to be processed. Default value is the
76	filename passed to the init function when initialised.
77
78	start_date : str
79	The first day of the retrieval period. Default value is None.
80
81	end_date :str
82	The last day of the retrieval period. Default value is None.
83
84	date_chunk : int
85	Length of period for a single mars retrieval. Default value is 3.
86
87	dtime :str
88	The time step in hours. Default value is None.
89
90	basetime : int
91	The time for a half day retrieval. The 12 hours upfront are to be
92	retrieved. Default value is None.
93
94	maxstep : int
95	The maximum forecast step for non flux data. Default value is None.
96
97	type : list of str
98	List of field type per retrieving hour. Default value is None.
99
100	time : list of str
101	List of retrieving times in hours. Default valuer is None.
102
103	step : list of str or str
104	List of forecast time steps in hours for non flux data.
105	Default value is None.
106
107	acctype : str
108	The field type for the accumulated forecast fields.
109	Default value is None.
110
111	acctime : str
112	The starting time of the accumulated forecasts. Default value is None.
113
114	accmaxstep : int
115	The maximum forecast step for the accumulated forecast fields
116	(flux data). Default value is None.
117
118	marsclass : str
119	Characterisation of dataset. Default value is None.
120
121	dataset : str
122	For public datasets there is the specific naming and parameter
123	dataset which has to be used to characterize the type of
124	data. Default value is None.
125
126	stream : str
127	Identifies the forecasting system used to generate the data.
128	Default value is None.
129
130	number : str
131	Selects the member in ensemble forecast run. Default value is 'OFF'.
132
133	expver : str
134	The version number of the dataset. Default value is '1'.
135
136	gaussian : str
137	This parameter is deprecated and should no longer be used.
138	Specifies the desired type of Gaussian grid for the output.
139	Default value is an empty string ''.
140
141	grid : str
142	Specifies the output grid which can be either a Gaussian grid
143	or a Latitude/Longitude grid. Default value is None.
144
145	area : str
146	Specifies the desired sub-area of data to be extracted.
147	Default value is None.
148
149	left : str
150	The western most longitude of the area to be extracted.
151	Default value is None.
152
153	lower : str
154	The southern most latitude of the area to be extracted.
155	Default value is None.
156
157	upper : str
158	The northern most latitued of the area to be extracted.
159	Default value is None.
160
161	right : str
162	The eastern most longitude of the area to be extracted.
163	Default value is None.
164
165	level : str
166	Specifies the maximum level. Default value is None.
167
168	levelist : str
169	Specifies the required level list. Default value is None.
170
171	resol : str
172	Specifies the desired triangular truncation of retrieved data,
173	before carrying out any other selected post-processing.
174	Default value is None.
175
176	gauss : int
177	Switch to select gaussian fields (1) or regular lat/lon (0).
178	Default value is 0.
179
180	accuracy : int
181	Specifies the number of bits per value to be used in the
182	generated GRIB coded fields. Default value is 24.
183
184	omega : int
185	Switch to select omega retrieval (1) or not (0). Default value is 0.
186
187	omegadiff : int
188	Switch to decide to calculate Omega and Dps/Dt from continuity
189	equation for diagnostic purposes (1) or not (0). Default value is 0.
190
191	eta : int
192	Switch to select direct retrieval of etadot from MARS (1) or
193	wether it has to be calculated (0). Then Default value is 0.
194
195	etadiff : int
196	Switch to select calculation of etadot and Dps/Dt from continuity
197	equation for diagnostic purposes (1) or not (0). Default value is 0.
198
199	etapar : int
200	GRIB parameter Id for etadot fields. Default value is 77.
201
202	dpdeta : int
203	Switch to select multiplication of etadot with dpdeta.
204	Default value is 1.
205
206	smooth : int
207	Spectral truncation of ETADOT after calculation on Gaussian grid.
208	Default value is 0.
209
210	format : str
211	The format of the GRIB data. Default value is 'GRIB1'.
212
213	addpar : str
214	List of additional surface level ECMWF parameter to be retrieved.
215	Default value is None.
216
217	prefix : str
218	Prefix string for the final FLEXPART/FLEXTRA ready input files.
219	Default value is 'EN'.
220
221	cwc : int
222	Switch to select wether the sum of cloud liquid water content and
223	cloud ice water content should be retrieved. Default value is 0.
224
225	wrf : int
226	Switch to select further parameters for retrievment to support
227	WRF simulations. Default value is 0.
228
229	ecfsdir : str
230	Path to the ECMWF storage 'ectmp:/${USER}/econdemand/'
231
232	mailfail : list of str
233	Email list for sending error log files from ECMWF servers.
234	The email addresses should be seperated by a comma.
235	Default value is ['${USER}'].
236
237	mailops : list of str
238	Email list for sending operational log files from ECMWF servers.
239	The email addresses should be seperated by a comma.
240	Default value is ['${USER}'].
241
242	ecstorage : int
243	Switch to select storage of FLEXPART ready output files
244	in the ECFS file system. Default value is 0.
245
246	ectrans : int
247	Switch to select the transfer of FLEXPART ready output files
248	to the gateway server. Default value is 0.
249
250	inputdir : str
251	Path to the temporary directory for the retrieval grib files and
252	other processing files. Default value is _config.PATH_INPUT_DIR.
253
254	outputdir : str
255	Path to the final directory where the final FLEXPART ready input
256	files are stored. Default value is None.
257
258	flexextractdir : str
259	Path to the flex_extract root directory. Default value is
260	_config.PATH_FLEXEXTRACT_DIR.
261
262	exedir : str
263	Path to the FORTRAN executable file. Default value is
264	_config.PATH_FORTRAN_SRC.
265
266	installdir : str
267	Path to a FLEXPART root directory. Default value is None.
268
269	makefile : str
270	Name of the makefile to be used for the Fortran program.
271	Default value is None.
272
273	destination : str
274	The remote destination which is used to transfer files
275	from ECMWF server to local gateway server. Default value is None.
276
277	gateway : str
278	The gateway server the user is using. Default value is None.
279
280	ecuid : str
281	The user id on ECMWF server. Default value is None.
282
283	ecgid : str
284	The group id on ECMWF server. Default value is None.
285
286	install_target : str
287	Defines the location where the installation is to be done.
288	Default value is None.
289
290	debug : int
291	Switch to keep temporary files at the end of postprocessing (1) or
292	to delete all temporary files except the final output files (0).
293	Default value is 0.
294
295	oper : int
296	Switch to prepare the operational job script. Start date, end date and
297	basetime will be prepared with environment variables.
298	Default value is 0.
299
300	request : int
301	Switch to select between just retrieving the data (0), writing the mars
302	parameter values to a csv file (1) or doing both (2).
303	Default value is 0.
304
305	public : int
306	Switch to select kind of ECMWF Web Api access and the
307	possible data sets. Public data sets (1) and Memberstate data sets (0).
308	Default value is 0.
309
310	ec_api : boolean
311	Tells wether the ECMWF Web API was able to load or not.
312	Default value is None.
313
314	cds_api : boolean
315	Tells wether the CDS API was able to load or not.
316	Default value is None.
317
318	purefc : int
319	Switch to decide wether the job is a pure forecast retrieval or
320	coupled with analysis data. Default value is 0.
321
322	rrint : int
323	Switch to select between old precipitation disaggregation method (0)
324	or the new IA3 disaggegration method (1). Default value is 0.
325
326	doubleelda : int
327	Switch to select the calculation of extra ensemble members for the
328	ELDA stream. It doubles the amount of retrieved ensemble members.
329
330	logicals : list of str
331	List of the names of logical switches which controls the flow
332	of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
333	'etadiff', 'dpdeta', 'cwc', 'wrf', 'ecstorage',
334	'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint', 'doubleelda']
335	'''
336
337	def __init__(self, filename):
338	'''Initialises the instance of ControlFile class and defines
339	all class attributes with default values. Afterwards calls
340	function __read_controlfile__ to read parameter from Control file.
341
342	Parameters
343	----------
344	filename : str
345	Name of CONTROL file.
346
347	Return
348	------
349
350	'''
351
352	# list of all possible class attributes and their default values
353	self.controlfile = filename
354	self.start_date = None
355	self.end_date = None
356	self.date_chunk = 3
357	self.job_chunk = None
358	self.dtime = None
359	self.basetime = None
360	self.maxstep = None
361	self.type = None
362	self.time = None
363	self.step = None
364	self.acctype = None
365	self.acctime = None
366	self.accmaxstep = None
367	self.marsclass = None
368	self.dataset = None
369	self.stream = None
370	self.number = 'OFF'
371	self.expver = '1'
372	self.gaussian = ''
373	self.grid = None
374	self.area = ''
375	self.left = None
376	self.lower = None
377	self.upper = None
378	self.right = None
379	self.level = None
380	self.levelist = None
381	self.resol = None
382	self.gauss = 0
383	self.accuracy = 24
384	self.omega = 0
385	self.omegadiff = 0
386	self.eta = 0
387	self.etadiff = 0
388	self.etapar = 77
389	self.dpdeta = 1
390	self.smooth = 0
391	self.format = 'GRIB1'
392	self.addpar = None
393	self.prefix = 'EN'
394	self.cwc = 0
395	self.wrf = 0
396	self.ecfsdir = 'ectmp:/${USER}/econdemand/'
397	self.mailfail = ['${USER}']
398	self.mailops = ['${USER}']
399	self.ecstorage = 0
400	self.ectrans = 0
401	self.inputdir = _config.PATH_INPUT_DIR
402	self.outputdir = None
403	self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR
404	self.exedir = _config.PATH_FORTRAN_SRC
405	self.installdir = None
406	self.makefile = None
407	self.destination = None
408	self.gateway = None
409	self.ecuid = None
410	self.ecgid = None
411	self.install_target = None
412	self.debug = 0
413	self.oper = 0
414	self.request = 0
415	self.public = 0
416	self.ec_api = None
417	self.cds_api = None
418	self.purefc = 0
419	self.rrint = 0
420	self.doubleelda = 0
421
422	self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
423	'dpdeta', 'cwc', 'wrf', 'ecstorage',
424	'ectrans', 'debug', 'oper', 'request', 'public',
425	'purefc', 'rrint', 'doubleelda']
426
427	self._read_controlfile()
428
429	return
430
431	def _read_controlfile(self):
432	'''Read CONTROL file and assign all CONTROL file variables.
433
434	Parameters
435	----------
436
437	Return
438	------
439
440	'''
441
442	try:
443	cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
444	with open(cfile) as f:
445	fdata = f.read().split('\n')
446	except IOError:
447	print('Could not read CONTROL file "' + cfile + '"')
448	print('Either it does not exist or its syntax is wrong.')
449	print('Try "' + sys.argv[0].split('/')[-1] + \
450	' -h" to print usage information')
451	sys.exit(1)
452
453	# go through every line and store parameter
454	for ldata in fdata:
455	if ldata and ldata[0] == '#':
456	# ignore comment line in control file
457	continue
458	if '#' in ldata:
459	# cut off comment
460	ldata = ldata.split('#')[0]
461	data = ldata.split()
462	if len(data) > 1:
463	if 'm_' in data[0].lower():
464	data[0] = data[0][2:]
465	if data[0].lower() == 'class':
466	data[0] = 'marsclass'
467	if data[0].lower() == 'day1':
468	data[0] = 'start_date'
469	if data[0].lower() == 'day2':
470	data[0] = 'end_date'
471	if len(data) == 2:
472	if '$' in data[1]:
473	setattr(self, data[0].lower(), data[1])
474	while '$' in data[1]:
475	i = data[1].index('$')
476	j = data[1].find('{')
477	k = data[1].find('}')
478	var = os.getenv(data[1][j+1:k])
479	if var is not None:
480	data[1] = data[1][:i] + var + data[1][k+1:]
481	else:
482	my_error('Could not find variable '
483	+ data[1][j+1:k] + ' while reading ' +
484	self.controlfile)
485	setattr(self, data[0].lower() + '_expanded', data[1])
486	else:
487	if data[1].lower() != 'none':
488	setattr(self, data[0].lower(), data[1])
489	else:
490	setattr(self, data[0].lower(), None)
491	elif len(data) > 2:
492	setattr(self, data[0].lower(), (data[1:]))
493	else:
494	pass
495
496	return
497
498	def __str__(self):
499	'''Prepares a string which have all the ControlFile class attributes
500	with its associated values. Each attribute is printed in one line and
501	in alphabetical order.
502
503	Example
504	-------
505	'age': 10
506	'color': 'Spotted'
507	'kids': 0
508	'legs': 2
509	'name': 'Dog'
510	'smell': 'Alot'
511
512	Parameters
513	----------
514
515	Return
516	------
517	string
518	Single string of concatenated ControlFile class attributes
519	with their values
520	'''
521	import collections
522
523	attrs = vars(self).copy()
524	attrs = collections.OrderedDict(sorted(attrs.items()))
525
526	return '\n'.join("%s: %s" % item for item in attrs.items())
527
528	def assign_args_to_control(self, args):
529	'''Overwrites the existing ControlFile instance attributes with
530	the command line arguments.
531
532	Parameters
533	----------
534	args : Namespace
535	Contains the commandline arguments from script/program call.
536
537	Return
538	------
539
540	'''
541
542	# get dictionary of command line parameters and eliminate all
543	# parameters which are None (were not specified)
544	args_dict = vars(args)
545	arguments = {k : args_dict[k] for k in args_dict
546	if args_dict[k] != None}
547
548	# assign all passed command line arguments to ControlFile instance
549	for k, v in arguments.items():
550	setattr(self, str(k), v)
551
552	return
553
554	def assign_envs_to_control(self, envs):
555	'''Assigns the ECMWF environment parameter.
556
557	Parameters
558	----------
559	envs : dict of str
560	Contains the ECMWF environment parameternames "ECUID", "ECGID",
561	"DESTINATION" and "GATEWAY" with its corresponding values.
562	They were read from the file "ECMWF_ENV".
563
564	Return
565	------
566
567	'''
568
569	for k, v in envs.items():
570	setattr(self, str(k).lower(), str(v))
571
572	return
573
574	def check_conditions(self, queue):
575	'''Checks a couple of necessary attributes and conditions,
576	such as if they exist and contain values.
577	Otherwise set default values.
578
579	Parameters
580	----------
581	queue : str
582	Name of the queue if submitted to the ECMWF servers.
583	Used to check if ecuid, ecgid, gateway and destination
584	are set correctly and are not empty.
585
586	Return
587	------
588
589	'''
590	check_logicals_type(self, self.logicals)
591
592	self.mailfail = check_mail(self.mailfail)
593
594	self.mailops = check_mail(self.mailops)
595
596	check_queue(queue, self.gateway, self.destination,
597	self.ecuid, self.ecgid)
598
599	self.outputdir, self.installdir = check_pathes(self.inputdir,
600	self.outputdir,
601	self.installdir,
602	self.flexextractdir)
603
604	self.start_date, self.end_date = check_dates(self.start_date,
605	self.end_date)
606
607	self.basetime = check_basetime(self.basetime)
608
609	self.levelist, self.level = check_levels(self.levelist, self.level)
610
611	self.step = check_step(self.step)
612
613	self.maxstep = check_maxstep(self.maxstep, self.step)
614
615	check_request(self.request,
616	os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
617
618	check_public(self.public, self.dataset, self.marsclass)
619
620	self.type = check_type(self.type, self.step)
621
622	self.time = check_time(self.time)
623
624	self.purefc = check_purefc(self.type)
625
626	self.type, self.time, self.step = check_len_type_time_step(self.type,
627	self.time,
628	self.step,
629	self.maxstep,
630	self.purefc)
631
632	self.acctype = check_acctype(self.acctype, self.type)
633
634	self.acctime = check_acctime(self.acctime, self.marsclass,
635	self.purefc, self.time)
636
637	self.accmaxstep = check_accmaxstep(self.accmaxstep, self.marsclass,
638	self.purefc, self.maxstep)
639
640	self.grid = check_grid(self.grid)
641
642	self.area = check_area(self.grid, self.area, self.upper, self.lower,
643	self.left, self.right)
644
645	self.addpar = check_addpar(self.addpar)
646
647	self.job_chunk = check_job_chunk(self.job_chunk)
648
649	self.number = check_number(self.number)
650
651	return
652
653	def to_list(self):
654	'''Just generates a list of strings containing the attributes and
655	assigned values except the attributes "_expanded", "exedir",
656	"flexextractdir" and "installdir".
657
658	Parameters
659	----------
660
661	Return
662	------
663	l : list of *
664	A sorted list of the all ControlFile class attributes with
665	their values except the attributes "_expanded", "exedir",
666	"flexextractdir" and "installdir".
667	'''
668
669	import collections
670
671	attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
672
673	l = list()
674
675	for item in attrs.items():
676	if '_expanded' in item[0]:
677	pass
678	elif 'exedir' in item[0]:
679	pass
680	elif 'installdir' in item[0]:
681	pass
682	elif 'flexextractdir' in item[0]:
683	pass
684	else:
685	if isinstance(item[1], list):
686	stot = ''
687	for s in item[1]:
688	stot += s + ' '
689
690	l.append("%s %s\n" % (item[0], stot))
691	else:
692	l.append("%s %s\n" % item)
693
694	return sorted(l)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: