Context Navigation

source: flex_extract.git/source/python/classes/ControlFile.py @ ae2756e

ctbtodev

Last change on this file since ae2756e was ae2756e, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
added a check for the number parameter
Property mode set to `100644`
File size: 22.6 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Leopold Haimberger (University of Vienna)
5	#
6	# @Date: November 2015
7	#
8	# @Change History:
9	#
10	# February 2018 - Anne Philipp (University of Vienna):
11	# - applied PEP8 style guide
12	# - added documentation
13	# - applied some minor modifications in programming style/structure
14	# - changed name of class Control to ControlFile for more
15	# self-explanation naming
16	# - outsource of class ControlFile
17	# - initialisation of class attributes ( to avoid high number of
18	# conditional statements and set default values )
19	# - divided assignment of attributes and the check of conditions
20	# - outsourced the commandline argument assignments to control attributes
21	#
22	# @License:
23	# (C) Copyright 2014-2019.
24	# Anne Philipp, Leopold Haimberger
25	#
26	# This work is licensed under the Creative Commons Attribution 4.0
27	# International License. To view a copy of this license, visit
28	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
29	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
30	#
31	# @Class Methods:
32	# __init__
33	# _read_controlfile
34	# __str__
35	# assign_args_to_control
36	# assign_envs_to_control
37	# check_conditions
38	# check_install_conditions
39	# to_list
40	#*******************************************************************************
41
42	# ------------------------------------------------------------------------------
43	# MODULES
44	# ------------------------------------------------------------------------------
45	import os
46	import re
47	import sys
48	import inspect
49	import numpy as np
50
51	# software specific classes and modules from flex_extract
52	sys.path.append('../')
53	import _config
54	from mods.tools import my_error, silent_remove
55	from mods.checks import (check_grid, check_area, check_levels, check_purefc,
56	check_step, check_mail, check_queue, check_pathes,
57	check_dates, check_maxstep, check_type, check_request,
58	check_basetime, check_public, check_acctype,
59	check_acctime, check_accmaxstep, check_time,
60	check_logicals_type, check_len_type_time_step,
61	check_addpar, check_job_chunk, check_number)
62
63	# ------------------------------------------------------------------------------
64	# CLASS
65	# ------------------------------------------------------------------------------
66	class ControlFile(object):
67	'''
68	Contains the information which are stored in the CONTROL files.
69
70	The CONTROL file is the steering part of the FLEXPART extraction
71	software. All necessary parameters needed to retrieve the data fields
72	from the MARS archive for driving FLEXPART are set in a CONTROL file.
73	Some specific parameters like the start and end dates can be overwritten
74	by the command line parameters, but in generel all parameters needed
75	for a complete set of fields for FLEXPART can be set in the CONTROL file.
76
77	Attributes
78	----------
79	controlfile : str
80	The name of the control file to be processed. Default value is the
81	filename passed to the init function when initialised.
82
83	start_date : str
84	The first day of the retrieval period. Default value is None.
85
86	end_date :str
87	The last day of the retrieval period. Default value is None.
88
89	date_chunk : int
90	Length of period for a single mars retrieval. Default value is 3.
91
92	dtime :str
93	The time step in hours. Default value is None.
94
95	basetime : int
96	The time for a half day retrieval. The 12 hours upfront are to be
97	retrieved. Default value is None.
98
99	maxstep : int
100	The maximum forecast step for non flux data. Default value is None.
101
102	type : list of str
103	List of field type per retrieving hour. Default value is None.
104
105	time : list of str
106	List of retrieving times in hours. Default valuer is None.
107
108	step : list of str or str
109	List of forecast time steps in hours for non flux data.
110	Default value is None.
111
112	acctype : str
113	The field type for the accumulated forecast fields.
114	Default value is None.
115
116	acctime : str
117	The starting time of the accumulated forecasts. Default value is None.
118
119	accmaxstep : int
120	The maximum forecast step for the accumulated forecast fields
121	(flux data). Default value is None.
122
123	marsclass : str
124	Characterisation of dataset. Default value is None.
125
126	dataset : str
127	For public datasets there is the specific naming and parameter
128	dataset which has to be used to characterize the type of
129	data. Default value is None.
130
131	stream : str
132	Identifies the forecasting system used to generate the data.
133	Default value is None.
134
135	number : str
136	Selects the member in ensemble forecast run. Default value is 'OFF'.
137
138	expver : str
139	The version number of the dataset. Default value is '1'.
140
141	gaussian : str
142	This parameter is deprecated and should no longer be used.
143	Specifies the desired type of Gaussian grid for the output.
144	Default value is an empty string ''.
145
146	grid : str
147	Specifies the output grid which can be either a Gaussian grid
148	or a Latitude/Longitude grid. Default value is None.
149
150	area : str
151	Specifies the desired sub-area of data to be extracted.
152	Default value is None.
153
154	left : str
155	The western most longitude of the area to be extracted.
156	Default value is None.
157
158	lower : str
159	The southern most latitude of the area to be extracted.
160	Default value is None.
161
162	upper : str
163	The northern most latitued of the area to be extracted.
164	Default value is None.
165
166	right : str
167	The eastern most longitude of the area to be extracted.
168	Default value is None.
169
170	level : str
171	Specifies the maximum level. Default value is None.
172
173	levelist : str
174	Specifies the required level list. Default value is None.
175
176	resol : str
177	Specifies the desired triangular truncation of retrieved data,
178	before carrying out any other selected post-processing.
179	Default value is None.
180
181	gauss : int
182	Switch to select gaussian fields (1) or regular lat/lon (0).
183	Default value is 0.
184
185	accuracy : int
186	Specifies the number of bits per value to be used in the
187	generated GRIB coded fields. Default value is 24.
188
189	omega : int
190	Switch to select omega retrieval (1) or not (0). Default value is 0.
191
192	omegadiff : int
193	Switch to decide to calculate Omega and Dps/Dt from continuity
194	equation for diagnostic purposes (1) or not (0). Default value is 0.
195
196	eta : int
197	Switch to select direct retrieval of etadot from MARS (1) or
198	wether it has to be calculated (0). Then Default value is 0.
199
200	etadiff : int
201	Switch to select calculation of etadot and Dps/Dt from continuity
202	equation for diagnostic purposes (1) or not (0). Default value is 0.
203
204	etapar : int
205	GRIB parameter Id for etadot fields. Default value is 77.
206
207	dpdeta : int
208	Switch to select multiplication of etadot with dpdeta.
209	Default value is 1.
210
211	smooth : int
212	Spectral truncation of ETADOT after calculation on Gaussian grid.
213	Default value is 0.
214
215	format : str
216	The format of the GRIB data. Default value is 'GRIB1'.
217
218	addpar : str
219	List of additional surface level ECMWF parameter to be retrieved.
220	Default value is None.
221
222	prefix : str
223	Prefix string for the final FLEXPART/FLEXTRA ready input files.
224	Default value is 'EN'.
225
226	cwc : int
227	Switch to select wether the sum of cloud liquid water content and
228	cloud ice water content should be retrieved. Default value is 0.
229
230	wrf : int
231	Switch to select further parameters for retrievment to support
232	WRF simulations. Default value is 0.
233
234	ecfsdir : str
235	Path to the ECMWF storage 'ectmp:/${USER}/econdemand/'
236
237	mailfail : list of str
238	Email list for sending error log files from ECMWF servers.
239	The email addresses should be seperated by a comma.
240	Default value is ['${USER}'].
241
242	mailops : list of str
243	Email list for sending operational log files from ECMWF servers.
244	The email addresses should be seperated by a comma.
245	Default value is ['${USER}'].
246
247	grib2flexpart : int 0
248	Switch to select generation of preprocessed FLEXPART files ".fp".
249	If it is selected, the program grib2flexpart will try
250	to convert the flex_extract output files into ".fp" format.
251
252	ecstorage : int
253	Switch to select storage of FLEXPART ready output files
254	in the ECFS file system. Default value is 0.
255
256	ectrans : int
257	Switch to select the transfer of FLEXPART ready output files
258	to the gateway server. Default value is 0.
259
260	inputdir : str
261	Path to the temporary directory for the retrieval grib files and
262	other processing files. Default value is _config.PATH_INPUT_DIR.
263
264	outputdir : str
265	Path to the final directory where the final FLEXPART ready input
266	files are stored. Default value is None.
267
268	flexextractdir : str
269	Path to the flex_extract root directory. Default value is
270	_config.PATH_FLEXEXTRACT_DIR.
271
272	exedir : str
273	Path to the FORTRAN executable file. Default value is
274	_config.PATH_FORTRAN_SRC.
275
276	flexpartdir : str
277	Path to a FLEXPART root directory. Default value is None.
278
279	makefile : str
280	Name of the makefile to be used for the Fortran program.
281	Default value is 'Makefile.gfortran'.
282
283	destination : str
284	The remote destination which is used to transfer files
285	from ECMWF server to local gateway server. Default value is None.
286
287	gateway : str
288	The gateway server the user is using. Default value is None.
289
290	ecuid : str
291	The user id on ECMWF server. Default value is None.
292
293	ecgid : str
294	The group id on ECMWF server. Default value is None.
295
296	install_target : str
297	Defines the location where the installation is to be done.
298	Default value is None.
299
300	debug : int
301	Switch to keep temporary files at the end of postprocessing (1) or
302	to delete all temporary files except the final output files (0).
303	Default value is 0.
304
305	oper : int
306	Switch to prepare the operational job script. Start date, end date and
307	basetime will be prepared with environment variables.
308	Default value is 0.
309
310	request : int
311	Switch to select between just retrieving the data (0), writing the mars
312	parameter values to a csv file (1) or doing both (2).
313	Default value is 0.
314
315	public : int
316	Switch to select kind of ECMWF Web Api access and the
317	possible data sets. Public data sets (1) and Memberstate data sets (0).
318	Default value is 0.
319
320	ecapi : boolean
321	Tells wether the ECMWF Web APi was able to load or not.
322	Default value is None.
323
324	purefc : int
325	Switch to decide wether the job is a pure forecast retrieval or
326	coupled with analysis data. Default value is 0.
327
328	rrint: int
329	Switch to select between old precipitation disaggregation method (0)
330	or the new IA3 disaggegration method (1). Default value is 0.
331
332	logicals : list of str
333	List of the names of logical switches which controls the flow
334	of the program. Default list is ['gauss', 'omega', 'omegadiff', 'eta',
335	'etadiff', 'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
336	'ectrans', 'debug', 'request', 'public', 'purefc', 'rrint']
337	'''
338
339	def __init__(self, filename):
340	'''Initialises the instance of ControlFile class and defines
341	all class attributes with default values. Afterwards calls
342	function __read_controlfile__ to read parameter from Control file.
343
344	Parameters
345	----------
346	filename : str
347	Name of CONTROL file.
348
349	Return
350	------
351
352	'''
353
354	# list of all possible class attributes and their default values
355	self.controlfile = filename
356	self.start_date = None
357	self.end_date = None
358	self.date_chunk = 3
359	self.job_chunk = None
360	self.dtime = None
361	self.basetime = None
362	self.maxstep = None
363	self.type = None
364	self.time = None
365	self.step = None
366	self.acctype = None
367	self.acctime = None
368	self.accmaxstep = None
369	self.marsclass = None
370	self.dataset = None
371	self.stream = None
372	self.number = 'OFF'
373	self.expver = '1'
374	self.gaussian = ''
375	self.grid = None
376	self.area = ''
377	self.left = None
378	self.lower = None
379	self.upper = None
380	self.right = None
381	self.level = None
382	self.levelist = None
383	self.resol = None
384	self.gauss = 0
385	self.accuracy = 24
386	self.omega = 0
387	self.omegadiff = 0
388	self.eta = 0
389	self.etadiff = 0
390	self.etapar = 77
391	self.dpdeta = 1
392	self.smooth = 0
393	self.format = 'GRIB1'
394	self.addpar = None
395	self.prefix = 'EN'
396	self.cwc = 0
397	self.wrf = 0
398	self.ecfsdir = 'ectmp:/${USER}/econdemand/'
399	self.mailfail = ['${USER}']
400	self.mailops = ['${USER}']
401	self.grib2flexpart = 0
402	self.ecstorage = 0
403	self.ectrans = 0
404	self.inputdir = _config.PATH_INPUT_DIR
405	self.outputdir = None
406	self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR
407	self.exedir = _config.PATH_FORTRAN_SRC
408	self.flexpartdir = None
409	self.makefile = 'Makefile.gfortran'
410	self.destination = None
411	self.gateway = None
412	self.ecuid = None
413	self.ecgid = None
414	self.install_target = None
415	self.debug = 0
416	self.oper = 0
417	self.request = 0
418	self.public = 0
419	self.ecapi = None
420	self.purefc = 0
421	self.rrint = 0
422
423	self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
424	'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
425	'ectrans', 'debug', 'oper', 'request', 'public',
426	'purefc', 'rrint']
427
428	self._read_controlfile()
429
430	return
431
432	def _read_controlfile(self):
433	'''Read CONTROL file and assign all CONTROL file variables.
434
435	Parameters
436	----------
437
438	Return
439	------
440
441	'''
442
443	try:
444	cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
445	with open(cfile) as f:
446	fdata = f.read().split('\n')
447	except IOError:
448	print('Could not read CONTROL file "' + cfile + '"')
449	print('Either it does not exist or its syntax is wrong.')
450	print('Try "' + sys.argv[0].split('/')[-1] + \
451	' -h" to print usage information')
452	sys.exit(1)
453
454	# go through every line and store parameter
455	for ldata in fdata:
456	if ldata and ldata[0] == '#':
457	# ignore comment line in control file
458	continue
459	if '#' in ldata:
460	# cut off comment
461	ldata = ldata.split('#')[0]
462	data = ldata.split()
463	if len(data) > 1:
464	if 'm_' in data[0].lower():
465	data[0] = data[0][2:]
466	if data[0].lower() == 'class':
467	data[0] = 'marsclass'
468	if data[0].lower() == 'day1':
469	data[0] = 'start_date'
470	if data[0].lower() == 'day2':
471	data[0] = 'end_date'
472	if len(data) == 2:
473	if '$' in data[1]:
474	setattr(self, data[0].lower(), data[1])
475	while '$' in data[1]:
476	i = data[1].index('$')
477	j = data[1].find('{')
478	k = data[1].find('}')
479	var = os.getenv(data[1][j+1:k])
480	if var is not None:
481	data[1] = data[1][:i] + var + data[1][k+1:]
482	else:
483	my_error(self.mailfail,
484	'Could not find variable '
485	+ data[1][j+1:k] + ' while reading ' +
486	self.controlfile)
487	setattr(self, data[0].lower() + '_expanded', data[1])
488	else:
489	if data[1].lower() != 'none':
490	setattr(self, data[0].lower(), data[1])
491	else:
492	setattr(self, data[0].lower(), None)
493	elif len(data) > 2:
494	setattr(self, data[0].lower(), (data[1:]))
495	else:
496	pass
497
498	return
499
500	def __str__(self):
501	'''Prepares a string which have all the ControlFile class attributes
502	with its associated values. Each attribute is printed in one line and
503	in alphabetical order.
504
505	Example
506	-------
507	'age': 10
508	'color': 'Spotted'
509	'kids': 0
510	'legs': 2
511	'name': 'Dog'
512	'smell': 'Alot'
513
514	Parameters
515	----------
516
517	Return
518	------
519	string
520	Single string of concatenated ControlFile class attributes
521	with their values
522	'''
523	import collections
524
525	attrs = vars(self).copy()
526	attrs = collections.OrderedDict(sorted(attrs.items()))
527
528	return '\n'.join("%s: %s" % item for item in attrs.items())
529
530	def assign_args_to_control(self, args):
531	'''Overwrites the existing ControlFile instance attributes with
532	the command line arguments.
533
534	Parameters
535	----------
536	args : Namespace
537	Contains the commandline arguments from script/program call.
538
539	Return
540	------
541
542	'''
543
544	# get dictionary of command line parameters and eliminate all
545	# parameters which are None (were not specified)
546	args_dict = vars(args)
547	arguments = {k : args_dict[k] for k in args_dict
548	if args_dict[k] != None}
549
550	# assign all passed command line arguments to ControlFile instance
551	for k, v in arguments.iteritems():
552	setattr(self, str(k), v)
553
554	return
555
556	def assign_envs_to_control(self, envs):
557	'''Assigns the ECMWF environment parameter.
558
559	Parameters
560	----------
561	envs : dict of str
562	Contains the ECMWF environment parameternames "ECUID", "ECGID",
563	"DESTINATION" and "GATEWAY" with its corresponding values.
564	They were read from the file "ECMWF_ENV".
565
566	Return
567	------
568
569	'''
570
571	for k, v in envs.iteritems():
572	setattr(self, str(k).lower(), str(v))
573
574	return
575
576	def check_conditions(self, queue):
577	'''Checks a couple of necessary attributes and conditions,
578	such as if they exist and contain values.
579	Otherwise set default values.
580
581	Parameters
582	----------
583	queue : str
584	Name of the queue if submitted to the ECMWF servers.
585	Used to check if ecuid, ecgid, gateway and destination
586	are set correctly and are not empty.
587
588	Return
589	------
590
591	'''
592	check_logicals_type(self, self.logicals)
593
594	self.mailfail = check_mail(self.mailfail)
595
596	self.mailops = check_mail(self.mailops)
597
598	check_queue(queue, self.gateway, self.destination,
599	self.ecuid, self.ecgid)
600
601	self.outputdir, self.flexpartdir = check_pathes(self.inputdir,
602	self.outputdir, self.flexpartdir, self.flexextractdir)
603
604	self.start_date, self.end_date = check_dates(self.start_date,
605	self.end_date)
606
607	self.basetime = check_basetime(self.basetime)
608
609	self.levelist, self.level = check_levels(self.levelist, self.level)
610
611	self.step = check_step(self.step, self.mailfail)
612
613	self.maxstep = check_maxstep(self.maxstep, self.step)
614
615	check_request(self.request,
616	os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
617
618	check_public(self.public, self.dataset)
619
620	self.type = check_type(self.type, self.step)
621
622	self.time = check_time(self.time)
623
624	self.type, self.time, self.step = check_len_type_time_step(self.type,
625	self.time,
626	self.step,
627	self.maxstep,
628	self.purefc)
629
630	self.acctype = check_acctype(self.acctype, self.type)
631
632	self.acctime = check_acctime(self.acctime, self.marsclass, self.purefc)
633
634	self.accmaxstep = check_accmaxstep(self.accmaxstep, self.marsclass,
635	self.purefc, self.maxstep)
636
637	self.purefc = check_purefc(self.type)
638
639	self.grid = check_grid(self.grid)
640
641	self.area = check_area(self.grid, self.area, self.upper, self.lower,
642	self.left, self.right)
643
644	self.addpar = check_addpar(self.addpar)
645
646	self.job_chunk = check_job_chunk(self.job_chunk)
647
648	self.number = check_number(self.number, self.mailfail)
649
650	return
651
652	def to_list(self):
653	'''Just generates a list of strings containing the attributes and
654	assigned values except the attributes "_expanded", "exedir",
655	"flexextractdir" and "flexpartdir".
656
657	Parameters
658	----------
659
660	Return
661	------
662	l : list of *
663	A sorted list of the all ControlFile class attributes with
664	their values except the attributes "_expanded", "exedir",
665	"flexextractdir" and "flexpartdir".
666	'''
667
668	import collections
669
670	attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
671
672	l = list()
673
674	for item in attrs.items():
675	if '_expanded' in item[0]:
676	pass
677	elif 'exedir' in item[0]:
678	pass
679	elif 'flexpartdir' in item[0]:
680	pass
681	elif 'flexextractdir' in item[0]:
682	pass
683	else:
684	if isinstance(item[1], list):
685	stot = ''
686	for s in item[1]:
687	stot += s + ' '
688
689	l.append("%s %s\n" % (item[0], stot))
690	else:
691	l.append("%s %s\n" % item)
692
693	return sorted(l)
694

Note: See TracBrowser for help on using the repository browser.

Download in other formats: