Context Navigation

source: flex_extract.git/source/python/mods/tools.py @ 2d56c04

ctbtodev

Last change on this file since 2d56c04 was 2d56c04, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
automatic detection of grid and area component formats (1/1000 or normal degree format)
Property mode set to `100644`
File size: 19.5 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Anne Philipp (University of Vienna)
5	#
6	# @Date: May 2018
7	#
8	# @Change History:
9	# October 2014 - Anne Fouilloux (University of Oslo)
10	# - created functions silent_remove and product (taken from ECMWF)
11	#
12	# November 2015 - Leopold Haimberger (University of Vienna)
13	# - created functions: interpret_args_and_control, clean_up
14	# my_error, normal_exit, init128, to_param_id
15	#
16	# April 2018 - Anne Philipp (University of Vienna):
17	# - applied PEP8 style guide
18	# - added documentation
19	# - moved all functions from file Flexparttools to this file tools
20	# - added function get_list_as_string
21	# - seperated args and control interpretation
22	#
23	# @License:
24	# (C) Copyright 2014-2018.
25	#
26	# This software is licensed under the terms of the Apache Licence Version 2.0
27	# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
28	#
29	# @Modul Description:
30	# This module contains a couple of helpful functions which are
31	# used in different places in flex_extract.
32	#
33	# @Module Content:
34	# - get_cmdline_arguments
35	# - clean_up
36	# - my_error
37	# - normal_exit
38	# - product
39	# - silent_remove
40	# - init128
41	# - to_param_id
42	# - get_list_as_string
43	# - make_dir
44	#
45	#*******************************************************************************
46
47	# ------------------------------------------------------------------------------
48	# MODULES
49	# ------------------------------------------------------------------------------
50	import os
51	import errno
52	import sys
53	import glob
54	import subprocess
55	import traceback
56	import exceptions
57	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
58
59	# ------------------------------------------------------------------------------
60	# FUNCTIONS
61	# ------------------------------------------------------------------------------
62
63	def none_or_str(value):
64	'''Converts the input string into pythons None-type if the string
65	contains string "None".
66
67	Parameters
68	----------
69	value : :obj:`string`
70	String to be checked for the "None" word.
71
72	Return
73	------
74	None or value:
75	Return depends on the content of the input value. If it was "None",
76	then the python type None is returned. Otherwise the string itself.
77	'''
78	if value == 'None':
79	return None
80	return value
81
82	def none_or_int(value):
83	'''Converts the input string into pythons None-type if the string
84	contains string "None". Otherwise it is converted to an integer value.
85
86	Parameters
87	----------
88	value : :obj:`string`
89	String to be checked for the "None" word.
90
91	Return
92	------
93	None or int(value):
94	Return depends on the content of the input value. If it was "None",
95	then the python type None is returned. Otherwise the string is
96	converted into an integer value.
97	'''
98	if value == 'None':
99	return None
100	return int(value)
101
102	def check_filepattern(filename):
103	'''
104	'''
105	if '.ksh' in filename:
106	return True
107	return False
108
109	def get_cmdline_arguments():
110	'''Decomposes the command line arguments and assigns them to variables.
111	Apply default values for non mentioned arguments.
112
113	Parameters
114	----------
115
116	Return
117	------
118	args : :obj:`Namespace`
119	Contains the commandline arguments from script/program call.
120	'''
121
122	parser = ArgumentParser(description='Retrieve FLEXPART input from \
123	ECMWF MARS archive',
124	formatter_class=ArgumentDefaultsHelpFormatter)
125
126	# the most important arguments
127	parser.add_argument("--start_date", dest="start_date",
128	type=none_or_str, default=None,
129	help="start date YYYYMMDD")
130	parser.add_argument("--end_date", dest="end_date",
131	type=none_or_str, default=None,
132	help="end_date YYYYMMDD")
133	parser.add_argument("--date_chunk", dest="date_chunk",
134	type=none_or_int, default=None,
135	help="# of days to be retrieved at once")
136	parser.add_argument("--controlfile", dest="controlfile",
137	type=none_or_str, default='CONTROL.temp',
138	help="file with CONTROL parameters")
139
140	# parameter for extra output information
141	parser.add_argument("--debug", dest="debug",
142	type=none_or_int, default=None,
143	help="debug mode - leave temporary files intact")
144	parser.add_argument("--request", dest="request",
145	type=none_or_int, default=None,
146	help="list all mars request in file mars_requests.dat \
147	and skip submission to mars")
148	parser.add_argument("--public", dest="public",
149	type=none_or_int, default=None,
150	help="public mode - retrieves the public datasets")
151
152	# some arguments that override the default in the CONTROL file
153	parser.add_argument("--basetime", dest="basetime",
154	type=none_or_int, default=None,
155	help="base such as 00 or 12 (for half day retrievals)")
156	parser.add_argument("--step", dest="step",
157	type=none_or_str, default=None,
158	help="steps such as 00/to/48")
159	parser.add_argument("--levelist", dest="levelist",
160	type=none_or_str, default=None,
161	help="Vertical levels to be retrieved, e.g. 30/to/60")
162	parser.add_argument("--area", dest="area",
163	type=none_or_str, default=None,
164	help="area defined as north/west/south/east")
165
166	# set the working directories
167	parser.add_argument("--inputdir", dest="inputdir",
168	type=none_or_str, default=None,
169	help="root directory for storing intermediate files")
170	parser.add_argument("--outputdir", dest="outputdir",
171	type=none_or_str, default=None,
172	help="root directory for storing output files")
173	parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
174	type=none_or_str, default=None,
175	help="FLEXPART root directory (to find grib2flexpart \
176	and COMMAND file)\n Normally flex_extract resides in \
177	the scripts directory of the FLEXPART distribution")
178
179	# this is only used by prepare_flexpart.py to rerun a postprocessing step
180	parser.add_argument("--ppid", dest="ppid",
181	type=none_or_str, default=None,
182	help="specify parent process id for \
183	rerun of prepare_flexpart")
184
185	# arguments for job submission to ECMWF, only needed by submit.py
186	parser.add_argument("--job_template", dest='job_template',
187	type=none_or_str, default="job.temp",
188	help="job template file for submission to ECMWF")
189	parser.add_argument("--queue", dest="queue",
190	type=none_or_str, default=None,
191	help="queue for submission to ECMWF \
192	(e.g. ecgate or cca )")
193
194	args = parser.parse_args()
195
196	return args
197
198	def read_ecenv(filepath):
199	'''Reads the file into a dictionary where the key values are the parameter
200	names.
201
202	Parameters
203	----------
204	filepath : :obj:`string`
205	Path to file where the ECMWF environment parameters are stored.
206
207	Return
208	------
209	envs : :obj:`dictionary`
210	Contains the environment parameter ecuid, ecgid, gateway
211	and destination for ECMWF server environments.
212	'''
213	envs= {}
214	try:
215	with open(filepath, 'r') as f:
216	for line in f:
217	data = line.strip().split()
218	envs[str(data[0])] = str(data[1])
219	except OSError as e:
220	print('... ERROR CODE: ' + str(e.errno))
221	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
222
223	sys.exit('\n... Error occured while trying to read ECMWF_ENV '
224	'file: ' + str(filepath))
225
226	return envs
227
228	def clean_up(c):
229	'''Remove files from the intermediate directory (inputdir).
230
231	It keeps the final FLEXPART input files if program runs without
232	ECMWF Api and keywords "ectrans" or "ecstorage" are set to "1".
233
234	Parameters
235	----------
236	c : :obj:`ControlFile`
237	Contains all the parameters of CONTROL file and
238	command line.
239
240	Return
241	------
242
243	'''
244
245	print("... clean inputdir!")
246
247	cleanlist = glob.glob(os.path.join(c.inputdir, "*"))
248
249	if cleanlist:
250	for element in cleanlist:
251	if c.prefix not in element:
252	silent_remove(element)
253	if c.ecapi is False and (c.ectrans == 1 or c.ecstorage == 1):
254	silent_remove(element)
255	print("... done!")
256	else:
257	print("... nothing to clean!")
258
259	return
260
261
262	def my_error(users, message='ERROR'):
263	'''Prints a specified error message which can be passed to the function
264	before exiting the program.
265
266	Parameters
267	----------
268	user : :obj:`list` of :obj:`string`
269	Contains all email addresses which should be notified.
270	It might also contain just the ecmwf user name which wil trigger
271	mailing to the associated email address for this user.
272
273	message : :obj:`string`, optional
274	Error message. Default value is "ERROR".
275
276	Return
277	------
278
279	'''
280
281	trace = '\n'.join(traceback.format_stack())
282	full_message = message + '\n\n' + trace
283
284	print(full_message)
285
286	send_mail(users, 'ERROR', full_message)
287
288	sys.exit(1)
289
290	return
291
292
293	def send_mail(users, success_mode, message):
294	'''Prints a specific exit message which can be passed to the function.
295
296	Parameters
297	----------
298	users : :obj:`list` of :obj:`string`
299	Contains all email addresses which should be notified.
300	It might also contain just the ecmwf user name which wil trigger
301	mailing to the associated email address for this user.
302
303	success_mode : :obj:``string`
304	States the exit mode of the program to put into
305	the mail subject line.
306
307	message : :obj:`string`, optional
308	Message for exiting program. Default value is "Done!".
309
310	Return
311	------
312
313	'''
314
315	for user in users:
316	if '${USER}' in user:
317	user = os.getenv('USER')
318	try:
319	p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ' +
320	success_mode, os.path.expandvars(user)],
321	stdin=subprocess.PIPE,
322	stdout=subprocess.PIPE,
323	stderr=subprocess.PIPE,
324	bufsize=1)
325	pout = p.communicate(input=message + '\n\n')[0]
326	except ValueError as e:
327	print('... ERROR: ' + str(e))
328	sys.exit('... Email could not be sent!')
329	except OSError as e:
330	print('... ERROR CODE: ' + str(e.errno))
331	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
332	sys.exit('... Email could not be sent!')
333	else:
334	print('Email sent to ' + os.path.expandvars(user))
335
336	return
337
338
339	def normal_exit(message='Done!'):
340	'''Prints a specific exit message which can be passed to the function.
341
342	Parameters
343	----------
344	message : :obj:`string`, optional
345	Message for exiting program. Default value is "Done!".
346
347	Return
348	------
349
350	'''
351
352	print(str(message))
353
354	return
355
356
357	def product(args, *kwds):
358	'''Creates combinations of all passed arguments.
359
360	This method combines the single characters of the passed arguments
361	with each other. So that each character of each argument value
362	will be combined with each character of the other arguments as a tuple.
363
364	Note
365	----
366	This method is taken from an example at the ECMWF wiki website.
367	https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
368
369	Example
370	-------
371	product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
372
373	product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
374
375	Parameters
376	----------
377	\*args : :obj:`list` or :obj:`string`
378	Positional arguments (arbitrary number).
379
380	\\kwds : :obj:`dictionary`
381	Contains all the keyword arguments from \*args.
382
383	Return
384	------
385	prod : :obj:`tuple`
386	Return will be done with "yield". A tuple of combined arguments.
387	See example in description above.
388	'''
389	try:
390	pools = map(tuple, args) * kwds.get('repeat', 1)
391	result = [[]]
392	for pool in pools:
393	result = [x + [y] for x in result for y in pool]
394	for prod in result:
395	yield tuple(prod)
396	except TypeError as e:
397	sys.exit('... PRODUCT GENERATION FAILED!')
398
399	return
400
401
402	def silent_remove(filename):
403	'''Remove file if it exists.
404	The function does not fail if the file does not exist.
405
406	Parameters
407	----------
408	filename : :obj:`string`
409	The name of the file to be removed without notification.
410
411	Return
412	------
413
414	'''
415	try:
416	os.remove(filename)
417	except OSError as e:
418	# errno.ENOENT = no such file or directory
419	if e.errno == errno.ENOENT:
420	pass
421	else:
422	raise # re-raise exception if a different error occured
423
424	return
425
426
427	def init128(filepath):
428	'''Opens and reads the grib file with table 128 information.
429
430	Parameters
431	----------
432	filepath : :obj:`string`
433	Path to file of ECMWF grib table number 128.
434
435	Return
436	------
437	table128 : :obj:`dictionary`
438	Contains the ECMWF grib table 128 information.
439	The key is the parameter number and the value is the
440	short name of the parameter.
441	'''
442	table128 = dict()
443	try:
444	with open(filepath) as f:
445	fdata = f.read().split('\n')
446	except OSError as e:
447	print('... ERROR CODE: ' + str(e.errno))
448	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
449
450	sys.exit('\n... Error occured while trying to read parameter '
451	'table file: ' + str(filepath))
452	else:
453	for data in fdata:
454	if data[0] != '!':
455	table128[data[0:3]] = data[59:64].strip()
456
457	return table128
458
459
460	def to_param_id(pars, table):
461	'''Transform parameter names to parameter ids with ECMWF grib table 128.
462
463	Parameters
464	----------
465	pars : :obj:`string`
466	Addpar argument from CONTROL file in the format of
467	parameter names instead of ids. The parameter short
468	names are sepearted with "/" and they are passed as
469	one single string.
470
471	table : :obj:`dictionary`
472	Contains the ECMWF grib table 128 information.
473	The key is the parameter number and the value is the
474	short name of the parameter.
475
476	Return
477	------
478	ipar : :obj:`list` of :obj:`integer`
479	List of addpar parameters from CONTROL file transformed to
480	parameter ids in the format of integer.
481	'''
482	if not pars:
483	return []
484	if not isinstance(pars, str):
485	pars=str(pars)
486
487	cpar = pars.upper().split('/')
488	ipar = []
489	for par in cpar:
490	for k, v in table.iteritems():
491	if par == k or par == v:
492	ipar.append(int(k))
493	break
494	else:
495	print('Warning: par ' + par + ' not found in table 128')
496
497	return ipar
498
499	def get_list_as_string(list_obj, concatenate_sign=', '):
500	'''Converts a list of arbitrary content into a single string.
501
502	Parameters
503	----------
504	list_obj : :obj:`list`
505	A list with arbitrary content.
506
507	concatenate_sign : :obj:`string`, optional
508	A string which is used to concatenate the single
509	list elements. Default value is ", ".
510
511	Return
512	------
513	str_of_list : :obj:`string`
514	The content of the list as a single string.
515	'''
516
517	if not isinstance(list_obj, list):
518	list_obj = list(list_obj)
519	str_of_list = concatenate_sign.join(str(l) for l in list_obj)
520
521	return str_of_list
522
523	def make_dir(directory):
524	'''Creates a directory.
525
526	It gives a warning if the directory already exists and skips process.
527	The program stops only if there is another problem.
528
529	Parameters
530	----------
531	directory : :obj:`string`
532	The path to directory which should be created.
533
534	Return
535	------
536
537	'''
538	try:
539	os.makedirs(directory)
540	except OSError as e:
541	# errno.EEXIST = directory already exists
542	if e.errno == errno.EEXIST:
543	print('WARNING: Directory {0} already exists!'.format(directory))
544	else:
545	raise # re-raise exception if a different error occured
546
547	return
548
549	def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
550	'''Uses the ecaccess-file-put command to send a file to the ECMWF servers.
551
552	Note
553	----
554	The return value is just for testing reasons. It does not have
555	to be used from the calling function since the whole error handling
556	is done in here.
557
558	Parameters
559	----------
560	ecd : :obj:`string`
561	The path were the file is stored.
562
563	filename : :obj:`string`
564	The name of the file to send to the ECMWF server.
565
566	target : :obj:`string`
567	The target queue where the file should be sent to.
568
569	ecuid : :obj:`string`
570	The user id on ECMWF server.
571
572	ecgid : :obj:`string`
573	The group id on ECMWF server.
574
575	Return
576	------
577
578	'''
579
580	try:
581	subprocess.check_output(['ecaccess-file-put',
582	ecd + '/' + filename,
583	target + ':/home/ms/' +
584	ecgid + '/' + ecuid +
585	'/' + filename],
586	stderr=subprocess.STDOUT)
587	except subprocess.CalledProcessError as e:
588	print('... ERROR CODE: ' + str(e.returncode))
589	print('... ERROR MESSAGE:\n \t ' + str(e))
590
591	print('\n... Do you have a valid ecaccess certification key?')
592	sys.exit('... ECACCESS-FILE-PUT FAILED!')
593	except OSError as e:
594	print('... ERROR CODE: ' + str(e.errno))
595	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
596
597	print('\n... Most likely the ECACCESS library is not available!')
598	sys.exit('... ECACCESS-FILE-PUT FAILED!')
599
600	return
601
602	def submit_job_to_ecserver(target, jobname):
603	'''Uses ecaccess-job-submit command to submit a job to the ECMWF server.
604
605	Note
606	----
607	The return value is just for testing reasons. It does not have
608	to be used from the calling function since the whole error handling
609	is done in here.
610
611	Parameters
612	----------
613	target : :obj:`string`
614	The target where the file should be sent to, e.g. the queue.
615
616	jobname : :obj:`string`
617	The name of the jobfile to be submitted to the ECMWF server.
618
619	Return
620	------
621	job_id : :obj:`int`
622	The id number of the job as a reference at the ecmwf server.
623	'''
624
625	try:
626	job_id = subprocess.check_output(['ecaccess-job-submit', '-queueName',
627	target, jobname])
628
629	except subprocess.CalledProcessError as e:
630	print('... ERROR CODE: ' + str(e.returncode))
631	print('... ERROR MESSAGE:\n \t ' + str(e))
632
633	print('\n... Do you have a valid ecaccess certification key?')
634	sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
635	except OSError as e:
636	print('... ERROR CODE: ' + str(e.errno))
637	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
638
639	print('\n... Most likely the ECACCESS library is not available!')
640	sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
641
642	return job_id

Note: See TracBrowser for help on using the repository browser.

Download in other formats: