Context Navigation

source: flex_extract.git/source/python/classes/ControlFile.py @ 90809b7

ctbtodev

Last change on this file since 90809b7 was 90809b7, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
python2 downgrade / changed flexpartdir to installdir / added purefc check /
Property mode set to `100644`
File size: 23.0 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - applied some minor modifications in programming style/structure
14	# - changed name of class Control to ControlFile for more
15	# self-explanation naming
16	# - outsource of class ControlFile
17	# - initialisation of class attributes ( to avoid high number of
18	# conditional statements and set default values )
19	# - divided assignment of attributes and the check of conditions
20	# - outsourced the commandline argument assignments to control attributes
21	#
22	# @License:
23	# (C) Copyright 2014-2019.
24	# Anne Philipp, Leopold Haimberger
25	#
26	# This work is licensed under the Creative Commons Attribution 4.0
27	# International License. To view a copy of this license, visit
28	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
29	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
30	#
31	# @Class Methods:
32	# __init__
33	# _read_controlfile
34	# __str__
35	# assign_args_to_control
36	# assign_envs_to_control
37	# check_conditions
38	# check_install_conditions
39	# to_list
40	#*******************************************************************************
41
42	# ------------------------------------------------------------------------------
43	# MODULES
44	# ------------------------------------------------------------------------------
45	from __future__ import print_function
46
47	import os
48	import re
49	import sys
50	import inspect
51
52	# software specific classes and modules from flex_extract
53	sys.path.append('../')
54	import _config
55	from mods.tools import my_error, silent_remove
56	from mods.checks import (check_grid, check_area, check_levels, check_purefc,
57	check_step, check_mail, check_queue, check_pathes,
58	check_dates, check_maxstep, check_type, check_request,
59	check_basetime, check_public, check_acctype,
60	check_acctime, check_accmaxstep, check_time,
61	check_logicals_type, check_len_type_time_step,
62	check_addpar, check_job_chunk, check_number)
63
64	# ------------------------------------------------------------------------------
65	# CLASS
66	# ------------------------------------------------------------------------------
67	class ControlFile(object):
68	'''
69	Contains the information which are stored in the CONTROL files.
70
71	The CONTROL file is the steering part of the FLEXPART extraction
72	software. All necessary parameters needed to retrieve the data fields
73	from the MARS archive for driving FLEXPART are set in a CONTROL file.
74	Some specific parameters like the start and end dates can be overwritten
75	by the command line parameters, but in generel all parameters needed
76	for a complete set of fields for FLEXPART can be set in the CONTROL file.
77
78	Attributes
79	----------
80	controlfile : str
81	The name of the control file to be processed. Default value is the
82	filename passed to the init function when initialised.
83
84	start_date : str
85	The first day of the retrieval period. Default value is None.
86
87	end_date :str
88	The last day of the retrieval period. Default value is None.
89
90	date_chunk : int
91	Length of period for a single mars retrieval. Default value is 3.
92
93	dtime :str
94	The time step in hours. Default value is None.
95
96	basetime : int
97	The time for a half day retrieval. The 12 hours upfront are to be
98	retrieved. Default value is None.
99
100	maxstep : int
101	The maximum forecast step for non flux data. Default value is None.
102
103	type : list of str
104	List of field type per retrieving hour. Default value is None.
105
106	time : list of str
107	List of retrieving times in hours. Default valuer is None.
108
109	step : list of str or str
110	List of forecast time steps in hours for non flux data.
111	Default value is None.
112
113	acctype : str
114	The field type for the accumulated forecast fields.
115	Default value is None.
116
117	acctime : str
118	The starting time of the accumulated forecasts. Default value is None.
119
120	accmaxstep : int
121	The maximum forecast step for the accumulated forecast fields
122	(flux data). Default value is None.
123
124	marsclass : str
125	Characterisation of dataset. Default value is None.
126
127	dataset : str
128	For public datasets there is the specific naming and parameter
129	dataset which has to be used to characterize the type of
130	data. Default value is None.
131
132	stream : str
133	Identifies the forecasting system used to generate the data.
134	Default value is None.
135
136	number : str
137	Selects the member in ensemble forecast run. Default value is 'OFF'.
138
139	expver : str
140	The version number of the dataset. Default value is '1'.
141
142	gaussian : str
143	This parameter is deprecated and should no longer be used.
144	Specifies the desired type of Gaussian grid for the output.
145	Default value is an empty string ''.
146
147	grid : str
148	Specifies the output grid which can be either a Gaussian grid
149	or a Latitude/Longitude grid. Default value is None.
150
151	area : str
152	Specifies the desired sub-area of data to be extracted.
153	Default value is None.
154
155	left : str
156	The western most longitude of the area to be extracted.
157	Default value is None.
158
159	lower : str
160	The southern most latitude of the area to be extracted.
161	Default value is None.
162
163	upper : str
164	The northern most latitued of the area to be extracted.
165	Default value is None.
166
167	right : str
168	The eastern most longitude of the area to be extracted.
169	Default value is None.
170
171	level : str
172	Specifies the maximum level. Default value is None.
173
174	levelist : str
175	Specifies the required level list. Default value is None.
176
177	resol : str
178	Specifies the desired triangular truncation of retrieved data,
179	before carrying out any other selected post-processing.
180	Default value is None.
181
182	gauss : int
183	Switch to select gaussian fields (1) or regular lat/lon (0).
184	Default value is 0.
185
186	accuracy : int
187	Specifies the number of bits per value to be used in the
188	generated GRIB coded fields. Default value is 24.
189
190	omega : int
191	Switch to select omega retrieval (1) or not (0). Default value is 0.
192
193	omegadiff : int
194	Switch to decide to calculate Omega and Dps/Dt from continuity
195	equation for diagnostic purposes (1) or not (0). Default value is 0.
196
197	eta : int
198	Switch to select direct retrieval of etadot from MARS (1) or
199	wether it has to be calculated (0). Then Default value is 0.
200
201	etadiff : int
202	Switch to select calculation of etadot and Dps/Dt from continuity
203	equation for diagnostic purposes (1) or not (0). Default value is 0.
204
205	etapar : int
206	GRIB parameter Id for etadot fields. Default value is 77.
207
208	dpdeta : int
209	Switch to select multiplication of etadot with dpdeta.
210	Default value is 1.
211
212	smooth : int
213	Spectral truncation of ETADOT after calculation on Gaussian grid.
214	Default value is 0.
215
216	format : str
217	The format of the GRIB data. Default value is 'GRIB1'.
218
219	addpar : str
220	List of additional surface level ECMWF parameter to be retrieved.
221	Default value is None.
222
223	prefix : str
224	Prefix string for the final FLEXPART/FLEXTRA ready input files.
225	Default value is 'EN'.
226
227	cwc : int
228	Switch to select wether the sum of cloud liquid water content and
229	cloud ice water content should be retrieved. Default value is 0.
230
231	wrf : int
232	Switch to select further parameters for retrievment to support
233	WRF simulations. Default value is 0.
234
235	ecfsdir : str
236	Path to the ECMWF storage 'ectmp:/${USER}/econdemand/'
237
238	mailfail : list of str
239	Email list for sending error log files from ECMWF servers.
240	The email addresses should be seperated by a comma.
241	Default value is ['${USER}'].
242
243	mailops : list of str
244	Email list for sending operational log files from ECMWF servers.
245	The email addresses should be seperated by a comma.
246	Default value is ['${USER}'].
247
248	grib2flexpart : int 0
249	Switch to select generation of preprocessed FLEXPART files ".fp".
250	If it is selected, the program grib2flexpart will try
251	to convert the flex_extract output files into ".fp" format.
252
253	ecstorage : int
254	Switch to select storage of FLEXPART ready output files
255	in the ECFS file system. Default value is 0.
256
257	ectrans : int
258	Switch to select the transfer of FLEXPART ready output files
259	to the gateway server. Default value is 0.
260
261	inputdir : str
262	Path to the temporary directory for the retrieval grib files and
263	other processing files. Default value is _config.PATH_INPUT_DIR.
264
265	outputdir : str
266	Path to the final directory where the final FLEXPART ready input
267	files are stored. Default value is None.
268
269	flexextractdir : str
270	Path to the flex_extract root directory. Default value is
271	_config.PATH_FLEXEXTRACT_DIR.
272
273	exedir : str
274	Path to the FORTRAN executable file. Default value is
275	_config.PATH_FORTRAN_SRC.
276
277	installdir : str
278	Path to a FLEXPART root directory. Default value is None.
279
280	makefile : str
281	Name of the makefile to be used for the Fortran program.
282	Default value is 'Makefile.gfortran'.
283
284	destination : str
285	The remote destination which is used to transfer files
286	from ECMWF server to local gateway server. Default value is None.
287
288	gateway : str
289	The gateway server the user is using. Default value is None.
290
291	ecuid : str
292	The user id on ECMWF server. Default value is None.
293
294	ecgid : str
295	The group id on ECMWF server. Default value is None.
296
297	install_target : str
298	Defines the location where the installation is to be done.
299	Default value is None.
300
301	debug : int
302	Switch to keep temporary files at the end of postprocessing (1) or
303	to delete all temporary files except the final output files (0).
304	Default value is 0.
305
306	oper : int
307	Switch to prepare the operational job script. Start date, end date and
308	basetime will be prepared with environment variables.
309	Default value is 0.
310
311	request : int
312	Switch to select between just retrieving the data (0), writing the mars
313	parameter values to a csv file (1) or doing both (2).
314	Default value is 0.
315
316	public : int
317	Switch to select kind of ECMWF Web Api access and the
318	possible data sets. Public data sets (1) and Memberstate data sets (0).
319	Default value is 0.
320
321	ec_api : boolean
322	Tells wether the ECMWF Web API was able to load or not.
323	Default value is None.
324
325	cds_api : boolean
326	Tells wether the CDS API was able to load or not.
327	Default value is None.
328
329	purefc : int
330	Switch to decide wether the job is a pure forecast retrieval or
331	coupled with analysis data. Default value is 0.
332
333	rrint : int
334	Switch to select between old precipitation disaggregation method (0)
335	or the new IA3 disaggegration method (1). Default value is 0.
336
337	doubleelda : int
338	Switch to select the calculation of extra ensemble members for the
339	ELDA stream. It doubles the amount of retrieved ensemble members.
340
341	logicals : list of str
342	List of the names of logical switches which controls the flow
343	of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
344	'etadiff', 'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
345	'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint', 'doubleelda']
346	'''
347
348	def __init__(self, filename):
349	'''Initialises the instance of ControlFile class and defines
350	all class attributes with default values. Afterwards calls
351	function __read_controlfile__ to read parameter from Control file.
352
353	Parameters
354	----------
355	filename : str
356	Name of CONTROL file.
357
358	Return
359	------
360
361	'''
362
363	# list of all possible class attributes and their default values
364	self.controlfile = filename
365	self.start_date = None
366	self.end_date = None
367	self.date_chunk = 3
368	self.job_chunk = None
369	self.dtime = None
370	self.basetime = None
371	self.maxstep = None
372	self.type = None
373	self.time = None
374	self.step = None
375	self.acctype = None
376	self.acctime = None
377	self.accmaxstep = None
378	self.marsclass = None
379	self.dataset = None
380	self.stream = None
381	self.number = 'OFF'
382	self.expver = '1'
383	self.gaussian = ''
384	self.grid = None
385	self.area = ''
386	self.left = None
387	self.lower = None
388	self.upper = None
389	self.right = None
390	self.level = None
391	self.levelist = None
392	self.resol = None
393	self.gauss = 0
394	self.accuracy = 24
395	self.omega = 0
396	self.omegadiff = 0
397	self.eta = 0
398	self.etadiff = 0
399	self.etapar = 77
400	self.dpdeta = 1
401	self.smooth = 0
402	self.format = 'GRIB1'
403	self.addpar = None
404	self.prefix = 'EN'
405	self.cwc = 0
406	self.wrf = 0
407	self.ecfsdir = 'ectmp:/${USER}/econdemand/'
408	self.mailfail = ['${USER}']
409	self.mailops = ['${USER}']
410	self.grib2flexpart = 0
411	self.ecstorage = 0
412	self.ectrans = 0
413	self.inputdir = _config.PATH_INPUT_DIR
414	self.outputdir = None
415	self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR
416	self.exedir = _config.PATH_FORTRAN_SRC
417	self.installdir = None
418	self.makefile = 'Makefile.gfortran'
419	self.destination = None
420	self.gateway = None
421	self.ecuid = None
422	self.ecgid = None
423	self.install_target = None
424	self.debug = 0
425	self.oper = 0
426	self.request = 0
427	self.public = 0
428	self.ec_api = None
429	self.cds_api = None
430	self.purefc = 0
431	self.rrint = 0
432	self.doubleelda = 0
433
434	self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
435	'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
436	'ectrans', 'debug', 'oper', 'request', 'public',
437	'purefc', 'rrint', 'doubleelda']
438
439	self._read_controlfile()
440
441	return
442
443	def _read_controlfile(self):
444	'''Read CONTROL file and assign all CONTROL file variables.
445
446	Parameters
447	----------
448
449	Return
450	------
451
452	'''
453
454	try:
455	cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
456	with open(cfile) as f:
457	fdata = f.read().split('\n')
458	except IOError:
459	print('Could not read CONTROL file "' + cfile + '"')
460	print('Either it does not exist or its syntax is wrong.')
461	print('Try "' + sys.argv[0].split('/')[-1] + \
462	' -h" to print usage information')
463	sys.exit(1)
464
465	# go through every line and store parameter
466	for ldata in fdata:
467	if ldata and ldata[0] == '#':
468	# ignore comment line in control file
469	continue
470	if '#' in ldata:
471	# cut off comment
472	ldata = ldata.split('#')[0]
473	data = ldata.split()
474	if len(data) > 1:
475	if 'm_' in data[0].lower():
476	data[0] = data[0][2:]
477	if data[0].lower() == 'class':
478	data[0] = 'marsclass'
479	if data[0].lower() == 'day1':
480	data[0] = 'start_date'
481	if data[0].lower() == 'day2':
482	data[0] = 'end_date'
483	if len(data) == 2:
484	if '$' in data[1]:
485	setattr(self, data[0].lower(), data[1])
486	while '$' in data[1]:
487	i = data[1].index('$')
488	j = data[1].find('{')
489	k = data[1].find('}')
490	var = os.getenv(data[1][j+1:k])
491	if var is not None:
492	data[1] = data[1][:i] + var + data[1][k+1:]
493	else:
494	my_error('Could not find variable '
495	+ data[1][j+1:k] + ' while reading ' +
496	self.controlfile)
497	setattr(self, data[0].lower() + '_expanded', data[1])
498	else:
499	if data[1].lower() != 'none':
500	setattr(self, data[0].lower(), data[1])
501	else:
502	setattr(self, data[0].lower(), None)
503	elif len(data) > 2:
504	setattr(self, data[0].lower(), (data[1:]))
505	else:
506	pass
507
508	return
509
510	def __str__(self):
511	'''Prepares a string which have all the ControlFile class attributes
512	with its associated values. Each attribute is printed in one line and
513	in alphabetical order.
514
515	Example
516	-------
517	'age': 10
518	'color': 'Spotted'
519	'kids': 0
520	'legs': 2
521	'name': 'Dog'
522	'smell': 'Alot'
523
524	Parameters
525	----------
526
527	Return
528	------
529	string
530	Single string of concatenated ControlFile class attributes
531	with their values
532	'''
533	import collections
534
535	attrs = vars(self).copy()
536	attrs = collections.OrderedDict(sorted(attrs.items()))
537
538	return '\n'.join("%s: %s" % item for item in attrs.items())
539
540	def assign_args_to_control(self, args):
541	'''Overwrites the existing ControlFile instance attributes with
542	the command line arguments.
543
544	Parameters
545	----------
546	args : Namespace
547	Contains the commandline arguments from script/program call.
548
549	Return
550	------
551
552	'''
553
554	# get dictionary of command line parameters and eliminate all
555	# parameters which are None (were not specified)
556	args_dict = vars(args)
557	arguments = {k : args_dict[k] for k in args_dict
558	if args_dict[k] != None}
559
560	# assign all passed command line arguments to ControlFile instance
561	for k, v in arguments.items():
562	setattr(self, str(k), v)
563
564	return
565
566	def assign_envs_to_control(self, envs):
567	'''Assigns the ECMWF environment parameter.
568
569	Parameters
570	----------
571	envs : dict of str
572	Contains the ECMWF environment parameternames "ECUID", "ECGID",
573	"DESTINATION" and "GATEWAY" with its corresponding values.
574	They were read from the file "ECMWF_ENV".
575
576	Return
577	------
578
579	'''
580
581	for k, v in envs.items():
582	setattr(self, str(k).lower(), str(v))
583
584	return
585
586	def check_conditions(self, queue):
587	'''Checks a couple of necessary attributes and conditions,
588	such as if they exist and contain values.
589	Otherwise set default values.
590
591	Parameters
592	----------
593	queue : str
594	Name of the queue if submitted to the ECMWF servers.
595	Used to check if ecuid, ecgid, gateway and destination
596	are set correctly and are not empty.
597
598	Return
599	------
600
601	'''
602	check_logicals_type(self, self.logicals)
603
604	self.mailfail = check_mail(self.mailfail)
605
606	self.mailops = check_mail(self.mailops)
607
608	check_queue(queue, self.gateway, self.destination,
609	self.ecuid, self.ecgid)
610
611	self.outputdir, self.installdir = check_pathes(self.inputdir,
612	self.outputdir, self.installdir, self.flexextractdir)
613
614	self.start_date, self.end_date = check_dates(self.start_date,
615	self.end_date)
616
617	self.basetime = check_basetime(self.basetime)
618
619	self.levelist, self.level = check_levels(self.levelist, self.level)
620
621	self.step = check_step(self.step, self.mailfail)
622
623	self.maxstep = check_maxstep(self.maxstep, self.step)
624
625	check_request(self.request,
626	os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
627
628	check_public(self.public, self.dataset)
629
630	self.type = check_type(self.type, self.step)
631
632	self.time = check_time(self.time)
633
634	self.purefc = check_purefc(self.type)
635
636	self.type, self.time, self.step = check_len_type_time_step(self.type,
637	self.time,
638	self.step,
639	self.maxstep,
640	self.purefc)
641
642	self.acctype = check_acctype(self.acctype, self.type)
643
644	self.acctime = check_acctime(self.acctime, self.marsclass,
645	self.purefc, self.time)
646
647	self.accmaxstep = check_accmaxstep(self.accmaxstep, self.marsclass,
648	self.purefc, self.maxstep)
649
650	self.grid = check_grid(self.grid)
651
652	self.area = check_area(self.grid, self.area, self.upper, self.lower,
653	self.left, self.right)
654
655	self.addpar = check_addpar(self.addpar)
656
657	self.job_chunk = check_job_chunk(self.job_chunk)
658
659	self.number = check_number(self.number, self.mailfail)
660
661	return
662
663	def to_list(self):
664	'''Just generates a list of strings containing the attributes and
665	assigned values except the attributes "_expanded", "exedir",
666	"flexextractdir" and "installdir".
667
668	Parameters
669	----------
670
671	Return
672	------
673	l : list of *
674	A sorted list of the all ControlFile class attributes with
675	their values except the attributes "_expanded", "exedir",
676	"flexextractdir" and "installdir".
677	'''
678
679	import collections
680
681	attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
682
683	l = list()
684
685	for item in attrs.items():
686	if '_expanded' in item[0]:
687	pass
688	elif 'exedir' in item[0]:
689	pass
690	elif 'installdir' in item[0]:
691	pass
692	elif 'flexextractdir' in item[0]:
693	pass
694	else:
695	if isinstance(item[1], list):
696	stot = ''
697	for s in item[1]:
698	stot += s + ' '
699
700	l.append("%s %s\n" % (item[0], stot))
701	else:
702	l.append("%s %s\n" % item)
703
704	return sorted(l)
705

Note: See TracBrowser for help on using the repository browser.

Download in other formats: