Context Navigation

source: flex_extract.git/source/python/mods/tools.py @ 6f951ca

ctbtodev

Last change on this file since 6f951ca was 6f951ca, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
new style of docstring params and updates in docstrings
Property mode set to `100644`
File size: 24.2 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Anne Philipp (University of Vienna)
5	#
6	# @Date: May 2018
7	#
8	# @Change History:
9	# October 2014 - Anne Fouilloux (University of Oslo)
10	# - created functions silent_remove and product (taken from ECMWF)
11	#
12	# November 2015 - Leopold Haimberger (University of Vienna)
13	# - created functions: interpret_args_and_control, clean_up
14	# my_error, normal_exit, init128, to_param_id
15	#
16	# April - December 2018 - Anne Philipp (University of Vienna):
17	# - applied PEP8 style guide
18	# - added documentation
19	# - moved all non class methods from former file Flexparttools in here
20	# - seperated args and control interpretation
21	# - added functions get_list_as_string, read_ecenv, send_mail, make_dir,
22	# put_file_to_ecserver, submit_job_to_ecserver, get_informations,
23	# get_dimensions, execute_subprocess, none_or_int, none_or_str
24	#
25	# @License:
26	# (C) Copyright 2014-2019.
27	# Anne Philipp, Leopold Haimberger
28	#
29	# This work is licensed under the Creative Commons Attribution 4.0
30	# International License. To view a copy of this license, visit
31	# http://creativecommons.org/licenses/by/4.0/ or send a letter to
32	# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
33	#
34	# @Methods:
35	# none_or_str
36	# none_or_int
37	# get_cmdline_args
38	# read_ecenv
39	# clean_up
40	# my_error
41	# send_mail
42	# normal_exit
43	# product
44	# silent_remove
45	# init128
46	# to_param_id
47	# get_list_as_string
48	# make_dir
49	# put_file_to_ecserver
50	# submit_job_to_ecserver
51	# get_informations
52	# get_dimensions
53	# execute_subprocess
54	#*******************************************************************************
55	'''This module contains a collection of diverse tasks within flex_extract.
56	'''
57
58	# ------------------------------------------------------------------------------
59	# MODULES
60	# ------------------------------------------------------------------------------
61	import os
62	import errno
63	import sys
64	import glob
65	import subprocess
66	import traceback
67	import exceptions
68	from datetime import datetime
69	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
70
71	# ------------------------------------------------------------------------------
72	# METHODS
73	# ------------------------------------------------------------------------------
74
75	def none_or_str(value):
76	'''Converts the input string into pythons None-type if the string
77	contains string "None".
78
79	Parameters
80	----------
81	value : str
82	String to be checked for the "None" word.
83
84	Return
85	------
86	None or value:
87	Return depends on the content of the input value. If it was "None",
88	then the python type None is returned. Otherwise the string itself.
89	'''
90	if value == 'None':
91	return None
92	return value
93
94	def none_or_int(value):
95	'''Converts the input string into pythons None-type if the string
96	contains string "None". Otherwise it is converted to an integer value.
97
98	Parameters
99	----------
100	value : str
101	String to be checked for the "None" word.
102
103	Return
104	------
105	None or int(value):
106	Return depends on the content of the input value. If it was "None",
107	then the python type None is returned. Otherwise the string is
108	converted into an integer value.
109	'''
110	if value == 'None':
111	return None
112	return int(value)
113
114	def get_cmdline_args():
115	'''Decomposes the command line arguments and assigns them to variables.
116	Apply default values for non mentioned arguments.
117
118	Parameters
119	----------
120
121	Return
122	------
123	args : Namespace
124	Contains the commandline arguments from script/program call.
125	'''
126
127	parser = ArgumentParser(description='Retrieve FLEXPART input from \
128	ECMWF MARS archive',
129	formatter_class=ArgumentDefaultsHelpFormatter)
130
131	# control parameters that override control file values
132	parser.add_argument("--start_date", dest="start_date",
133	type=none_or_str, default=None,
134	help="start date YYYYMMDD")
135	parser.add_argument("--end_date", dest="end_date",
136	type=none_or_str, default=None,
137	help="end_date YYYYMMDD")
138	parser.add_argument("--date_chunk", dest="date_chunk",
139	type=none_or_int, default=None,
140	help="# of days to be retrieved at once")
141	parser.add_argument("--job_chunk", dest="job_chunk",
142	type=none_or_int, default=None,
143	help="# of days to be retrieved within a single job")
144	parser.add_argument("--controlfile", dest="controlfile",
145	type=none_or_str, default='CONTROL.temp',
146	help="file with CONTROL parameters")
147	parser.add_argument("--basetime", dest="basetime",
148	type=none_or_int, default=None,
149	help="base such as 00 or 12 (for half day retrievals)")
150	parser.add_argument("--step", dest="step",
151	type=none_or_str, default=None,
152	help="steps such as 00/to/48")
153	parser.add_argument("--levelist", dest="levelist",
154	type=none_or_str, default=None,
155	help="Vertical levels to be retrieved, e.g. 30/to/60")
156	parser.add_argument("--area", dest="area",
157	type=none_or_str, default=None,
158	help="area defined as north/west/south/east")
159
160	# some switches
161	parser.add_argument("--debug", dest="debug",
162	type=none_or_int, default=None,
163	help="debug mode - leave temporary files intact")
164	parser.add_argument("--request", dest="request",
165	type=none_or_int, default=None,
166	help="list all mars requests in file mars_requests.dat")
167	parser.add_argument("--public", dest="public",
168	type=none_or_int, default=None,
169	help="public mode - retrieves the public datasets")
170	parser.add_argument("--rrint", dest="rrint",
171	type=none_or_int, default=None,
172	help="select old or new precipitation interpolation \
173	0 - old method\
174	1 - new method (additional subgrid points)")
175
176	# set directories
177	parser.add_argument("--inputdir", dest="inputdir",
178	type=none_or_str, default=None,
179	help="root directory for storing intermediate files")
180	parser.add_argument("--outputdir", dest="outputdir",
181	type=none_or_str, default=None,
182	help="root directory for storing output files")
183	parser.add_argument("--flexpartdir", dest="flexpartdir",
184	type=none_or_str, default=None,
185	help="FLEXPART root directory (to find grib2flexpart \
186	and COMMAND file)\n Normally flex_extract resides in \
187	the scripts directory of the FLEXPART distribution")
188
189	# this is only used by prepare_flexpart.py to rerun a postprocessing step
190	parser.add_argument("--ppid", dest="ppid",
191	type=none_or_str, default=None,
192	help="specify parent process id for \
193	rerun of prepare_flexpart")
194
195	# arguments for job submission to ECMWF, only needed by submit.py
196	parser.add_argument("--job_template", dest='job_template',
197	type=none_or_str, default="job.temp",
198	help="job template file for submission to ECMWF")
199	parser.add_argument("--queue", dest="queue",
200	type=none_or_str, default=None,
201	help="queue for submission to ECMWF \
202	(e.g. ecgate or cca )")
203
204	args = parser.parse_args()
205
206	return args
207
208	def read_ecenv(filepath):
209	'''Reads the file into a dictionary where the key values are the parameter
210	names.
211
212	Parameters
213	----------
214	filepath : str
215	Path to file where the ECMWF environment parameters are stored.
216
217	Return
218	------
219	envs : dict
220	Contains the environment parameter ecuid, ecgid, gateway
221	and destination for ECMWF server environments.
222	'''
223	envs= {}
224	try:
225	with open(filepath, 'r') as f:
226	for line in f:
227	data = line.strip().split()
228	envs[str(data[0])] = str(data[1])
229	except OSError as e:
230	print('... ERROR CODE: ' + str(e.errno))
231	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
232
233	sys.exit('\n... Error occured while trying to read ECMWF_ENV '
234	'file: ' + str(filepath))
235
236	return envs
237
238	def clean_up(c):
239	'''Remove files from the intermediate directory (inputdir).
240
241	It keeps the final FLEXPART input files if program runs without
242	ECMWF Api and keywords "ectrans" or "ecstorage" are set to "1".
243
244	Parameters
245	----------
246	c : ControlFile
247	Contains all the parameters of CONTROL file and
248	command line.
249
250	Return
251	------
252
253	'''
254
255	print("... clean inputdir!")
256
257	cleanlist = glob.glob(os.path.join(c.inputdir, "*"))
258
259	if cleanlist:
260	for element in cleanlist:
261	if c.prefix not in element:
262	silent_remove(element)
263	if c.ecapi is False and (c.ectrans == 1 or c.ecstorage == 1):
264	silent_remove(element)
265	print("... done!")
266	else:
267	print("... nothing to clean!")
268
269	return
270
271
272	def my_error(users, message='ERROR'):
273	'''Prints a specified error message which can be passed to the function
274	before exiting the program.
275
276	Parameters
277	----------
278	user : list of str
279	Contains all email addresses which should be notified.
280	It might also contain just the ecmwf user name which wil trigger
281	mailing to the associated email address for this user.
282
283	message : str, optional
284	Error message. Default value is "ERROR".
285
286	Return
287	------
288
289	'''
290
291	trace = '\n'.join(traceback.format_stack())
292	full_message = message + '\n\n' + trace
293
294	print(full_message)
295
296	send_mail(users, 'ERROR', full_message)
297
298	sys.exit(1)
299
300	return
301
302
303	def send_mail(users, success_mode, message):
304	'''Prints a specific exit message which can be passed to the function.
305
306	Parameters
307	----------
308	users : list of str
309	Contains all email addresses which should be notified.
310	It might also contain just the ecmwf user name which wil trigger
311	mailing to the associated email address for this user.
312
313	success_mode : str
314	States the exit mode of the program to put into
315	the mail subject line.
316
317	message : str, optional
318	Message for exiting program. Default value is "Done!".
319
320	Return
321	------
322
323	'''
324
325	for user in users:
326	if '${USER}' in user:
327	user = os.getenv('USER')
328	try:
329	p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ' +
330	success_mode, os.path.expandvars(user)],
331	stdin=subprocess.PIPE,
332	stdout=subprocess.PIPE,
333	stderr=subprocess.PIPE,
334	bufsize=1)
335	pout = p.communicate(input=message + '\n\n')[0]
336	except ValueError as e:
337	print('... ERROR: ' + str(e))
338	sys.exit('... Email could not be sent!')
339	except OSError as e:
340	print('... ERROR CODE: ' + str(e.errno))
341	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
342	sys.exit('... Email could not be sent!')
343	else:
344	print('Email sent to ' + os.path.expandvars(user))
345
346	return
347
348
349	def normal_exit(message='Done!'):
350	'''Prints a specific exit message which can be passed to the function.
351
352	Parameters
353	----------
354	message : str, optional
355	Message for exiting program. Default value is "Done!".
356
357	Return
358	------
359
360	'''
361
362	print(str(message))
363
364	return
365
366
367	def product(args, *kwds):
368	'''Creates combinations of all passed arguments.
369
370	This method combines the single characters of the passed arguments
371	with each other. So that each character of each argument value
372	will be combined with each character of the other arguments as a tuple.
373
374	Note
375	----
376	This method is taken from an example at the ECMWF wiki website.
377	https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
378
379	Example
380	-------
381	product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
382
383	product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
384
385	Parameters
386	----------
387	\*args : list or str
388	Positional arguments (arbitrary number).
389
390	\\kwds : dict
391	Contains all the keyword arguments from \*args.
392
393	Return
394	------
395	prod : :obj:`tuple`
396	Return will be done with "yield". A tuple of combined arguments.
397	See example in description above.
398	'''
399	try:
400	pools = map(tuple, args) * kwds.get('repeat', 1)
401	result = [[]]
402	for pool in pools:
403	result = [x + [y] for x in result for y in pool]
404	for prod in result:
405	yield tuple(prod)
406	except TypeError as e:
407	sys.exit('... PRODUCT GENERATION FAILED!')
408
409	return
410
411
412	def silent_remove(filename):
413	'''Remove file if it exists.
414	The function does not fail if the file does not exist.
415
416	Parameters
417	----------
418	filename : str
419	The name of the file to be removed without notification.
420
421	Return
422	------
423
424	'''
425	try:
426	os.remove(filename)
427	except OSError as e:
428	# errno.ENOENT = no such file or directory
429	if e.errno == errno.ENOENT:
430	pass
431	else:
432	raise # re-raise exception if a different error occured
433
434	return
435
436
437	def init128(filepath):
438	'''Opens and reads the grib file with table 128 information.
439
440	Parameters
441	----------
442	filepath : str
443	Path to file of ECMWF grib table number 128.
444
445	Return
446	------
447	table128 : dict
448	Contains the ECMWF grib table 128 information.
449	The key is the parameter number and the value is the
450	short name of the parameter.
451	'''
452	table128 = dict()
453	try:
454	with open(filepath) as f:
455	fdata = f.read().split('\n')
456	except OSError as e:
457	print('... ERROR CODE: ' + str(e.errno))
458	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
459
460	sys.exit('\n... Error occured while trying to read parameter '
461	'table file: ' + str(filepath))
462	else:
463	for data in fdata:
464	if data[0] != '!':
465	table128[data[0:3]] = data[59:64].strip()
466
467	return table128
468
469
470	def to_param_id(pars, table):
471	'''Transform parameter names to parameter ids with ECMWF grib table 128.
472
473	Parameters
474	----------
475	pars : str
476	Addpar argument from CONTROL file in the format of
477	parameter names instead of ids. The parameter short
478	names are sepearted with "/" and they are passed as
479	one single string.
480
481	table : dict
482	Contains the ECMWF grib table 128 information.
483	The key is the parameter number and the value is the
484	short name of the parameter.
485
486	Return
487	------
488	ipar : list of int
489	List of addpar parameters from CONTROL file transformed to
490	parameter ids in the format of integer.
491	'''
492	if not pars:
493	return []
494	if not isinstance(pars, str):
495	pars=str(pars)
496
497	cpar = pars.upper().split('/')
498	ipar = []
499	for par in cpar:
500	for k, v in table.iteritems():
501	if par == k or par == v:
502	ipar.append(int(k))
503	break
504	else:
505	print('Warning: par ' + par + ' not found in table 128')
506
507	return ipar
508
509	def get_list_as_string(list_obj, concatenate_sign=', '):
510	'''Converts a list of arbitrary content into a single string.
511
512	Parameters
513	----------
514	list_obj : list of *
515	A list with arbitrary content.
516
517	concatenate_sign : str, optional
518	A string which is used to concatenate the single
519	list elements. Default value is ", ".
520
521	Return
522	------
523	str_of_list : str
524	The content of the list as a single string.
525	'''
526
527	if not isinstance(list_obj, list):
528	list_obj = list(list_obj)
529	str_of_list = concatenate_sign.join(str(l) for l in list_obj)
530
531	return str_of_list
532
533	def make_dir(directory):
534	'''Creates a directory.
535
536	It gives a warning if the directory already exists and skips process.
537	The program stops only if there is another problem.
538
539	Parameters
540	----------
541	directory : str
542	The path to directory which should be created.
543
544	Return
545	------
546
547	'''
548	try:
549	os.makedirs(directory)
550	except OSError as e:
551	# errno.EEXIST = directory already exists
552	if e.errno == errno.EEXIST:
553	print('WARNING: Directory {0} already exists!'.format(directory))
554	else:
555	raise # re-raise exception if a different error occured
556
557	return
558
559	def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
560	'''Uses the ecaccess-file-put command to send a file to the ECMWF servers.
561
562	Note
563	----
564	The return value is just for testing reasons. It does not have
565	to be used from the calling function since the whole error handling
566	is done in here.
567
568	Parameters
569	----------
570	ecd : str
571	The path were the file is stored.
572
573	filename : str
574	The name of the file to send to the ECMWF server.
575
576	target : str
577	The target queue where the file should be sent to.
578
579	ecuid : str
580	The user id on ECMWF server.
581
582	ecgid : str
583	The group id on ECMWF server.
584
585	Return
586	------
587
588	'''
589
590	try:
591	subprocess.check_output(['ecaccess-file-put',
592	ecd + '/' + filename,
593	target + ':/home/ms/' +
594	ecgid + '/' + ecuid +
595	'/' + filename],
596	stderr=subprocess.STDOUT)
597	except subprocess.CalledProcessError as e:
598	print('... ERROR CODE: ' + str(e.returncode))
599	print('... ERROR MESSAGE:\n \t ' + str(e))
600
601	print('\n... Do you have a valid ecaccess certification key?')
602	sys.exit('... ECACCESS-FILE-PUT FAILED!')
603	except OSError as e:
604	print('... ERROR CODE: ' + str(e.errno))
605	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
606
607	print('\n... Most likely the ECACCESS library is not available!')
608	sys.exit('... ECACCESS-FILE-PUT FAILED!')
609
610	return
611
612	def submit_job_to_ecserver(target, jobname):
613	'''Uses ecaccess-job-submit command to submit a job to the ECMWF server.
614
615	Note
616	----
617	The return value is just for testing reasons. It does not have
618	to be used from the calling function since the whole error handling
619	is done in here.
620
621	Parameters
622	----------
623	target : str
624	The target where the file should be sent to, e.g. the queue.
625
626	jobname : str
627	The name of the jobfile to be submitted to the ECMWF server.
628
629	Return
630	------
631	job_id : int
632	The id number of the job as a reference at the ecmwf server.
633	'''
634
635	try:
636	job_id = subprocess.check_output(['ecaccess-job-submit', '-queueName',
637	target, jobname])
638
639	except subprocess.CalledProcessError as e:
640	print('... ERROR CODE: ' + str(e.returncode))
641	print('... ERROR MESSAGE:\n \t ' + str(e))
642
643	print('\n... Do you have a valid ecaccess certification key?')
644	sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
645	except OSError as e:
646	print('... ERROR CODE: ' + str(e.errno))
647	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
648
649	print('\n... Most likely the ECACCESS library is not available!')
650	sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
651
652	return job_id
653
654
655	def get_informations(filename):
656	'''Gets basic information from an example grib file.
657
658	These information are important for later use and the
659	initialization of numpy arrays for data storing.
660
661	Parameters
662	----------
663	filename : str
664	Name of the file which will be opened to extract basic information.
665
666	Return
667	------
668	data : dict
669	Contains basic informations of the ECMWF grib files, e.g.
670	'Ni', 'Nj', 'latitudeOfFirstGridPointInDegrees',
671	'longitudeOfFirstGridPointInDegrees', 'latitudeOfLastGridPointInDegrees',
672	'longitudeOfLastGridPointInDegrees', 'jDirectionIncrementInDegrees',
673	'iDirectionIncrementInDegrees', 'missingValue'
674	'''
675	from eccodes import codes_grib_new_from_file, codes_get, codes_release
676
677	data = {}
678
679	# --- open file ---
680	print("Opening file for getting information data --- %s" % filename)
681	with open(filename) as f:
682	# load first message from file
683	gid = codes_grib_new_from_file(f)
684
685	# information needed from grib message
686	keys = [
687	'Ni',
688	'Nj',
689	'latitudeOfFirstGridPointInDegrees',
690	'longitudeOfFirstGridPointInDegrees',
691	'latitudeOfLastGridPointInDegrees',
692	'longitudeOfLastGridPointInDegrees',
693	'jDirectionIncrementInDegrees',
694	'iDirectionIncrementInDegrees',
695	'missingValue',
696	]
697
698	print('\nInformations are: ')
699	for key in keys:
700	# Get the value of the key in a grib message.
701	data[key] = codes_get(gid,key)
702	print("%s = %s" % (key,data[key]))
703
704	# Free the memory for the message referred as gribid.
705	codes_release(gid)
706
707	return data
708
709
710	def get_dimensions(info, purefc, dtime, index_vals, start_date, end_date):
711	'''This function specifies the correct dimensions for x, y and t.
712
713	Parameters
714	----------
715	info : dict
716	Contains basic informations of the ECMWF grib files, e.g.
717	'Ni', 'Nj', 'latitudeOfFirstGridPointInDegrees',
718	'longitudeOfFirstGridPointInDegrees', 'latitudeOfLastGridPointInDegrees',
719	'longitudeOfLastGridPointInDegrees', 'jDirectionIncrementInDegrees',
720	'iDirectionIncrementInDegrees', 'missingValue'
721
722	purefc : int
723	Switch for definition of pure forecast mode or not.
724
725	dtime : str
726	Time step in hours.
727
728	index_vals : list of list of str
729	Contains the values from the keys used for a distinct selection
730	of grib messages in processing the grib files.
731	Content looks like e.g.:
732	index_vals[0]: ('20171106', '20171107', '20171108') ; date
733	index_vals[1]: ('0', '1200', '1800', '600') ; time
734	index_vals[2]: ('0', '12', '3', '6', '9') ; stepRange
735
736	start_date : str
737	The start date of the retrieval job.
738
739	end_date : str
740	The end date of the retrieval job.
741
742	Return
743	------
744	(ix, jy, it) : tuple of int
745	Dimension in x-direction, y-direction and in time.
746	'''
747
748	ix = info['Ni']
749
750	jy = info['Nj']
751
752	if not purefc:
753	it = ((end_date - start_date).days + 1) * 24/int(dtime)
754	else:
755	# #no of step * #no of times * #no of days
756	it = len(index_vals[2]) * len(index_vals[1]) * len(index_vals[0])
757
758	return (ix, jy, it)
759
760
761	def execute_subprocess(cmd_list, error_msg='SUBPROCESS FAILED!'):
762	'''Executes a command line instruction via a subprocess.
763
764	Error handling is done if an error occures.
765
766	Parameters
767	----------
768	cmd_list : list of str
769	A list of the components for the command line execution. Each
770	list entry is a single part of the command which is seperated from
771	the rest by a blank space.
772	E.g. ['mv', file1, file2]
773
774	Return
775	------
776	error_msg : str, optional
777	The possible error message if the subprocess failed.
778	By default it will just tell "SUBPROCESS FAILED!".
779	'''
780
781	try:
782	subprocess.check_call(cmd_list)
783	except subprocess.CalledProcessError as e:
784	print('... ERROR CODE: ' + str(e.returncode))
785	print('... ERROR MESSAGE:\n \t ' + str(e))
786
787	sys.exit('... ' + error_msg)
788	except OSError as e:
789	print('... ERROR CODE: ' + str(e.errno))
790	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
791
792	sys.exit('... ' + error_msg)
793
794	return

Note: See TracBrowser for help on using the repository browser.

Download in other formats: