Context Navigation

source: flex_extract.git/source/python/mods/tools.py @ 97f4f4c

ctbtodev

Last change on this file since 97f4f4c was 97f4f4c, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
better check on grid and arae parameter formats
Property mode set to `100644`
File size: 19.4 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Anne Philipp (University of Vienna)
5	#
6	# @Date: May 2018
7	#
8	# @Change History:
9	# October 2014 - Anne Fouilloux (University of Oslo)
10	# - created functions silent_remove and product (taken from ECMWF)
11	#
12	# November 2015 - Leopold Haimberger (University of Vienna)
13	# - created functions: interpret_args_and_control, clean_up
14	# my_error, normal_exit, init128, to_param_id
15	#
16	# April 2018 - Anne Philipp (University of Vienna):
17	# - applied PEP8 style guide
18	# - added documentation
19	# - moved all functions from file Flexparttools to this file tools
20	# - added function get_list_as_string
21	# - seperated args and control interpretation
22	#
23	# @License:
24	# (C) Copyright 2014-2018.
25	#
26	# This software is licensed under the terms of the Apache Licence Version 2.0
27	# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
28	#
29	# @Modul Description:
30	# This module contains a couple of helpful functions which are
31	# used in different places in flex_extract.
32	#
33	# @Module Content:
34	# - get_cmdline_arguments
35	# - clean_up
36	# - my_error
37	# - normal_exit
38	# - product
39	# - silent_remove
40	# - init128
41	# - to_param_id
42	# - get_list_as_string
43	# - make_dir
44	#
45	#*******************************************************************************
46
47	# ------------------------------------------------------------------------------
48	# MODULES
49	# ------------------------------------------------------------------------------
50	import os
51	import errno
52	import sys
53	import glob
54	import subprocess
55	import traceback
56	import exceptions
57	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
58
59	# ------------------------------------------------------------------------------
60	# FUNCTIONS
61	# ------------------------------------------------------------------------------
62
63	def none_or_str(value):
64	'''Converts the input string into pythons None-type if the string
65	contains string "None".
66
67	Parameters
68	----------
69	value : :obj:`string`
70	String to be checked for the "None" word.
71
72	Return
73	------
74	None or value:
75	Return depends on the content of the input value. If it was "None",
76	then the python type None is returned. Otherwise the string itself.
77	'''
78	if value == 'None':
79	return None
80	return value
81
82	def none_or_int(value):
83	'''Converts the input string into pythons None-type if the string
84	contains string "None". Otherwise it is converted to an integer value.
85
86	Parameters
87	----------
88	value : :obj:`string`
89	String to be checked for the "None" word.
90
91	Return
92	------
93	None or int(value):
94	Return depends on the content of the input value. If it was "None",
95	then the python type None is returned. Otherwise the string is
96	converted into an integer value.
97	'''
98	if value == 'None':
99	return None
100	return int(value)
101
102	def get_cmdline_arguments():
103	'''Decomposes the command line arguments and assigns them to variables.
104	Apply default values for non mentioned arguments.
105
106	Parameters
107	----------
108
109	Return
110	------
111	args : :obj:`Namespace`
112	Contains the commandline arguments from script/program call.
113	'''
114
115	parser = ArgumentParser(description='Retrieve FLEXPART input from \
116	ECMWF MARS archive',
117	formatter_class=ArgumentDefaultsHelpFormatter)
118
119	# the most important arguments
120	parser.add_argument("--start_date", dest="start_date",
121	type=none_or_str, default=None,
122	help="start date YYYYMMDD")
123	parser.add_argument("--end_date", dest="end_date",
124	type=none_or_str, default=None,
125	help="end_date YYYYMMDD")
126	parser.add_argument("--date_chunk", dest="date_chunk",
127	type=none_or_int, default=None,
128	help="# of days to be retrieved at once")
129	parser.add_argument("--controlfile", dest="controlfile",
130	type=none_or_str, default='CONTROL.temp',
131	help="file with CONTROL parameters")
132
133	# parameter for extra output information
134	parser.add_argument("--debug", dest="debug",
135	type=none_or_int, default=None,
136	help="debug mode - leave temporary files intact")
137	parser.add_argument("--request", dest="request",
138	type=none_or_int, default=None,
139	help="list all mars request in file mars_requests.dat \
140	and skip submission to mars")
141	parser.add_argument("--public", dest="public",
142	type=none_or_int, default=None,
143	help="public mode - retrieves the public datasets")
144
145	# some arguments that override the default in the CONTROL file
146	parser.add_argument("--basetime", dest="basetime",
147	type=none_or_int, default=None,
148	help="base such as 00 or 12 (for half day retrievals)")
149	parser.add_argument("--step", dest="step",
150	type=none_or_str, default=None,
151	help="steps such as 00/to/48")
152	parser.add_argument("--levelist", dest="levelist",
153	type=none_or_str, default=None,
154	help="Vertical levels to be retrieved, e.g. 30/to/60")
155	parser.add_argument("--area", dest="area",
156	type=none_or_str, default=None,
157	help="area defined as north/west/south/east")
158
159	# set the working directories
160	parser.add_argument("--inputdir", dest="inputdir",
161	type=none_or_str, default=None,
162	help="root directory for storing intermediate files")
163	parser.add_argument("--outputdir", dest="outputdir",
164	type=none_or_str, default=None,
165	help="root directory for storing output files")
166	parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
167	type=none_or_str, default=None,
168	help="FLEXPART root directory (to find grib2flexpart \
169	and COMMAND file)\n Normally flex_extract resides in \
170	the scripts directory of the FLEXPART distribution")
171
172	# this is only used by prepare_flexpart.py to rerun a postprocessing step
173	parser.add_argument("--ppid", dest="ppid",
174	type=none_or_str, default=None,
175	help="specify parent process id for \
176	rerun of prepare_flexpart")
177
178	# arguments for job submission to ECMWF, only needed by submit.py
179	parser.add_argument("--job_template", dest='job_template',
180	type=none_or_str, default="job.temp",
181	help="job template file for submission to ECMWF")
182	parser.add_argument("--queue", dest="queue",
183	type=none_or_str, default=None,
184	help="queue for submission to ECMWF \
185	(e.g. ecgate or cca )")
186
187	args = parser.parse_args()
188
189	return args
190
191	def read_ecenv(filepath):
192	'''Reads the file into a dictionary where the key values are the parameter
193	names.
194
195	Parameters
196	----------
197	filepath : :obj:`string`
198	Path to file where the ECMWF environment parameters are stored.
199
200	Return
201	------
202	envs : :obj:`dictionary`
203	Contains the environment parameter ecuid, ecgid, gateway
204	and destination for ECMWF server environments.
205	'''
206	envs= {}
207	try:
208	with open(filepath, 'r') as f:
209	for line in f:
210	data = line.strip().split()
211	envs[str(data[0])] = str(data[1])
212	except OSError as e:
213	print('... ERROR CODE: ' + str(e.errno))
214	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
215
216	sys.exit('\n... Error occured while trying to read ECMWF_ENV '
217	'file: ' + str(filepath))
218
219	return envs
220
221	def clean_up(c):
222	'''Remove files from the intermediate directory (inputdir).
223
224	It keeps the final FLEXPART input files if program runs without
225	ECMWF Api and keywords "ectrans" or "ecstorage" are set to "1".
226
227	Parameters
228	----------
229	c : :obj:`ControlFile`
230	Contains all the parameters of CONTROL file and
231	command line.
232
233	Return
234	------
235
236	'''
237
238	print("... clean inputdir!")
239
240	cleanlist = glob.glob(os.path.join(c.inputdir, "*"))
241
242	if cleanlist:
243	for element in cleanlist:
244	if c.prefix not in element:
245	silent_remove(element)
246	if c.ecapi is False and (c.ectrans == 1 or c.ecstorage == 1):
247	silent_remove(element)
248	print("... done!")
249	else:
250	print("... nothing to clean!")
251
252	return
253
254
255	def my_error(users, message='ERROR'):
256	'''Prints a specified error message which can be passed to the function
257	before exiting the program.
258
259	Parameters
260	----------
261	user : :obj:`list` of :obj:`string`
262	Contains all email addresses which should be notified.
263	It might also contain just the ecmwf user name which wil trigger
264	mailing to the associated email address for this user.
265
266	message : :obj:`string`, optional
267	Error message. Default value is "ERROR".
268
269	Return
270	------
271
272	'''
273
274	trace = '\n'.join(traceback.format_stack())
275	full_message = message + '\n\n' + trace
276
277	print(full_message)
278
279	send_mail(users, 'ERROR', full_message)
280
281	sys.exit(1)
282
283	return
284
285
286	def send_mail(users, success_mode, message):
287	'''Prints a specific exit message which can be passed to the function.
288
289	Parameters
290	----------
291	users : :obj:`list` of :obj:`string`
292	Contains all email addresses which should be notified.
293	It might also contain just the ecmwf user name which wil trigger
294	mailing to the associated email address for this user.
295
296	success_mode : :obj:``string`
297	States the exit mode of the program to put into
298	the mail subject line.
299
300	message : :obj:`string`, optional
301	Message for exiting program. Default value is "Done!".
302
303	Return
304	------
305
306	'''
307
308	for user in users:
309	if '${USER}' in user:
310	user = os.getenv('USER')
311	try:
312	p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ' +
313	success_mode, os.path.expandvars(user)],
314	stdin=subprocess.PIPE,
315	stdout=subprocess.PIPE,
316	stderr=subprocess.PIPE,
317	bufsize=1)
318	pout = p.communicate(input=message + '\n\n')[0]
319	except ValueError as e:
320	print('... ERROR: ' + str(e))
321	sys.exit('... Email could not be sent!')
322	except OSError as e:
323	print('... ERROR CODE: ' + str(e.errno))
324	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
325	sys.exit('... Email could not be sent!')
326	else:
327	print('Email sent to ' + os.path.expandvars(user))
328
329	return
330
331
332	def normal_exit(message='Done!'):
333	'''Prints a specific exit message which can be passed to the function.
334
335	Parameters
336	----------
337	message : :obj:`string`, optional
338	Message for exiting program. Default value is "Done!".
339
340	Return
341	------
342
343	'''
344
345	print(str(message))
346
347	return
348
349
350	def product(args, *kwds):
351	'''Creates combinations of all passed arguments.
352
353	This method combines the single characters of the passed arguments
354	with each other. So that each character of each argument value
355	will be combined with each character of the other arguments as a tuple.
356
357	Note
358	----
359	This method is taken from an example at the ECMWF wiki website.
360	https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
361
362	Example
363	-------
364	product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
365
366	product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
367
368	Parameters
369	----------
370	\*args : :obj:`list` or :obj:`string`
371	Positional arguments (arbitrary number).
372
373	\\kwds : :obj:`dictionary`
374	Contains all the keyword arguments from \*args.
375
376	Return
377	------
378	prod : :obj:`tuple`
379	Return will be done with "yield". A tuple of combined arguments.
380	See example in description above.
381	'''
382	try:
383	pools = map(tuple, args) * kwds.get('repeat', 1)
384	result = [[]]
385	for pool in pools:
386	result = [x + [y] for x in result for y in pool]
387	for prod in result:
388	yield tuple(prod)
389	except TypeError as e:
390	sys.exit('... PRODUCT GENERATION FAILED!')
391
392	return
393
394
395	def silent_remove(filename):
396	'''Remove file if it exists.
397	The function does not fail if the file does not exist.
398
399	Parameters
400	----------
401	filename : :obj:`string`
402	The name of the file to be removed without notification.
403
404	Return
405	------
406
407	'''
408	try:
409	os.remove(filename)
410	except OSError as e:
411	# errno.ENOENT = no such file or directory
412	if e.errno == errno.ENOENT:
413	pass
414	else:
415	raise # re-raise exception if a different error occured
416
417	return
418
419
420	def init128(filepath):
421	'''Opens and reads the grib file with table 128 information.
422
423	Parameters
424	----------
425	filepath : :obj:`string`
426	Path to file of ECMWF grib table number 128.
427
428	Return
429	------
430	table128 : :obj:`dictionary`
431	Contains the ECMWF grib table 128 information.
432	The key is the parameter number and the value is the
433	short name of the parameter.
434	'''
435	table128 = dict()
436	try:
437	with open(filepath) as f:
438	fdata = f.read().split('\n')
439	except OSError as e:
440	print('... ERROR CODE: ' + str(e.errno))
441	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
442
443	sys.exit('\n... Error occured while trying to read parameter '
444	'table file: ' + str(filepath))
445	else:
446	for data in fdata:
447	if data[0] != '!':
448	table128[data[0:3]] = data[59:64].strip()
449
450	return table128
451
452
453	def to_param_id(pars, table):
454	'''Transform parameter names to parameter ids with ECMWF grib table 128.
455
456	Parameters
457	----------
458	pars : :obj:`string`
459	Addpar argument from CONTROL file in the format of
460	parameter names instead of ids. The parameter short
461	names are sepearted with "/" and they are passed as
462	one single string.
463
464	table : :obj:`dictionary`
465	Contains the ECMWF grib table 128 information.
466	The key is the parameter number and the value is the
467	short name of the parameter.
468
469	Return
470	------
471	ipar : :obj:`list` of :obj:`integer`
472	List of addpar parameters from CONTROL file transformed to
473	parameter ids in the format of integer.
474	'''
475	if not pars:
476	return []
477	if not isinstance(pars, str):
478	pars=str(pars)
479
480	cpar = pars.upper().split('/')
481	ipar = []
482	for par in cpar:
483	for k, v in table.iteritems():
484	if par == k or par == v:
485	ipar.append(int(k))
486	break
487	else:
488	print('Warning: par ' + par + ' not found in table 128')
489
490	return ipar
491
492	def get_list_as_string(list_obj, concatenate_sign=', '):
493	'''Converts a list of arbitrary content into a single string.
494
495	Parameters
496	----------
497	list_obj : :obj:`list`
498	A list with arbitrary content.
499
500	concatenate_sign : :obj:`string`, optional
501	A string which is used to concatenate the single
502	list elements. Default value is ", ".
503
504	Return
505	------
506	str_of_list : :obj:`string`
507	The content of the list as a single string.
508	'''
509
510	if not isinstance(list_obj, list):
511	list_obj = list(list_obj)
512	str_of_list = concatenate_sign.join(str(l) for l in list_obj)
513
514	return str_of_list
515
516	def make_dir(directory):
517	'''Creates a directory.
518
519	It gives a warning if the directory already exists and skips process.
520	The program stops only if there is another problem.
521
522	Parameters
523	----------
524	directory : :obj:`string`
525	The path to directory which should be created.
526
527	Return
528	------
529
530	'''
531	try:
532	os.makedirs(directory)
533	except OSError as e:
534	# errno.EEXIST = directory already exists
535	if e.errno == errno.EEXIST:
536	print('WARNING: Directory {0} already exists!'.format(directory))
537	else:
538	raise # re-raise exception if a different error occured
539
540	return
541
542	def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
543	'''Uses the ecaccess-file-put command to send a file to the ECMWF servers.
544
545	Note
546	----
547	The return value is just for testing reasons. It does not have
548	to be used from the calling function since the whole error handling
549	is done in here.
550
551	Parameters
552	----------
553	ecd : :obj:`string`
554	The path were the file is stored.
555
556	filename : :obj:`string`
557	The name of the file to send to the ECMWF server.
558
559	target : :obj:`string`
560	The target queue where the file should be sent to.
561
562	ecuid : :obj:`string`
563	The user id on ECMWF server.
564
565	ecgid : :obj:`string`
566	The group id on ECMWF server.
567
568	Return
569	------
570
571	'''
572
573	try:
574	subprocess.check_output(['ecaccess-file-put',
575	ecd + '/' + filename,
576	target + ':/home/ms/' +
577	ecgid + '/' + ecuid +
578	'/' + filename],
579	stderr=subprocess.STDOUT)
580	except subprocess.CalledProcessError as e:
581	print('... ERROR CODE: ' + str(e.returncode))
582	print('... ERROR MESSAGE:\n \t ' + str(e))
583
584	print('\n... Do you have a valid ecaccess certification key?')
585	sys.exit('... ECACCESS-FILE-PUT FAILED!')
586	except OSError as e:
587	print('... ERROR CODE: ' + str(e.errno))
588	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
589
590	print('\n... Most likely the ECACCESS library is not available!')
591	sys.exit('... ECACCESS-FILE-PUT FAILED!')
592
593	return
594
595	def submit_job_to_ecserver(target, jobname):
596	'''Uses ecaccess-job-submit command to submit a job to the ECMWF server.
597
598	Note
599	----
600	The return value is just for testing reasons. It does not have
601	to be used from the calling function since the whole error handling
602	is done in here.
603
604	Parameters
605	----------
606	target : :obj:`string`
607	The target where the file should be sent to, e.g. the queue.
608
609	jobname : :obj:`string`
610	The name of the jobfile to be submitted to the ECMWF server.
611
612	Return
613	------
614	job_id : :obj:`int`
615	The id number of the job as a reference at the ecmwf server.
616	'''
617
618	try:
619	job_id = subprocess.check_output(['ecaccess-job-submit', '-queueName',
620	target, jobname])
621
622	except subprocess.CalledProcessError as e:
623	print('... ERROR CODE: ' + str(e.returncode))
624	print('... ERROR MESSAGE:\n \t ' + str(e))
625
626	print('\n... Do you have a valid ecaccess certification key?')
627	sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
628	except OSError as e:
629	print('... ERROR CODE: ' + str(e.errno))
630	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
631
632	print('\n... Most likely the ECACCESS library is not available!')
633	sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
634
635	return job_id

Note: See TracBrowser for help on using the repository browser.

Download in other formats: