Context Navigation

source: flex_extract.git/source/python/mods/tools.py @ 274f9ef

ctbtodev

Last change on this file since 274f9ef was 274f9ef, checked in by Anne Philipp <anne.philipp@…>, 6 years ago
Converted docstrings to numpy style and build first structure for sphinxdocumentation (incl API)
Property mode set to `100644`
File size: 18.3 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Anne Philipp (University of Vienna)
5	#
6	# @Date: May 2018
7	#
8	# @Change History:
9	# October 2014 - Anne Fouilloux (University of Oslo)
10	# - created functions silent_remove and product (taken from ECMWF)
11	#
12	# November 2015 - Leopold Haimberger (University of Vienna)
13	# - created functions: interpret_args_and_control, clean_up
14	# my_error, normal_exit, init128, to_param_id
15	#
16	# April 2018 - Anne Philipp (University of Vienna):
17	# - applied PEP8 style guide
18	# - added documentation
19	# - moved all functions from file Flexparttools to this file tools
20	# - added function get_list_as_string
21	# - seperated args and control interpretation
22	#
23	# @License:
24	# (C) Copyright 2014-2018.
25	#
26	# This software is licensed under the terms of the Apache Licence Version 2.0
27	# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
28	#
29	# @Modul Description:
30	# This module contains a couple of helpful functions which are
31	# used in different places in flex_extract.
32	#
33	# @Module Content:
34	# - get_cmdline_arguments
35	# - clean_up
36	# - my_error
37	# - normal_exit
38	# - product
39	# - silent_remove
40	# - init128
41	# - to_param_id
42	# - get_list_as_string
43	# - make_dir
44	#
45	#*******************************************************************************
46
47	# ------------------------------------------------------------------------------
48	# MODULES
49	# ------------------------------------------------------------------------------
50	import os
51	import errno
52	import sys
53	import glob
54	import subprocess
55	import traceback
56	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
57
58	# ------------------------------------------------------------------------------
59	# FUNCTIONS
60	# ------------------------------------------------------------------------------
61
62	def none_or_str(value):
63	'''Converts the input string into pythons None-type if the string
64	contains string "None".
65
66	Parameters
67	----------
68	value : :obj:`string`
69	String to be checked for the "None" word.
70
71	Return
72	------
73	None or value:
74	Return depends on the content of the input value. If it was "None",
75	then the python type None is returned. Otherwise the string itself.
76	'''
77	if value == 'None':
78	return None
79	return value
80
81	def none_or_int(value):
82	'''Converts the input string into pythons None-type if the string
83	contains string "None". Otherwise it is converted to an integer value.
84
85	Parameters
86	----------
87	value : :obj:`string`
88	String to be checked for the "None" word.
89
90	Return
91	------
92	None or int(value):
93	Return depends on the content of the input value. If it was "None",
94	then the python type None is returned. Otherwise the string is
95	converted into an integer value.
96	'''
97	if value == 'None':
98	return None
99	return int(value)
100
101	def get_cmdline_arguments():
102	'''Decomposes the command line arguments and assigns them to variables.
103	Apply default values for non mentioned arguments.
104
105	Parameters
106	----------
107
108	Return
109	------
110	args : :obj:`Namespace`
111	Contains the commandline arguments from script/program call.
112	'''
113
114	parser = ArgumentParser(description='Retrieve FLEXPART input from \
115	ECMWF MARS archive',
116	formatter_class=ArgumentDefaultsHelpFormatter)
117
118	# the most important arguments
119	parser.add_argument("--start_date", dest="start_date",
120	type=none_or_str, default=None,
121	help="start date YYYYMMDD")
122	parser.add_argument("--end_date", dest="end_date",
123	type=none_or_str, default=None,
124	help="end_date YYYYMMDD")
125	parser.add_argument("--date_chunk", dest="date_chunk",
126	type=none_or_int, default=None,
127	help="# of days to be retrieved at once")
128	parser.add_argument("--controlfile", dest="controlfile",
129	type=none_or_str, default='CONTROL.temp',
130	help="file with CONTROL parameters")
131
132	# parameter for extra output information
133	parser.add_argument("--debug", dest="debug",
134	type=none_or_int, default=None,
135	help="debug mode - leave temporary files intact")
136	parser.add_argument("--request", dest="request",
137	type=none_or_int, default=None,
138	help="list all mars request in file mars_requests.dat \
139	and skip submission to mars")
140	parser.add_argument("--public", dest="public",
141	type=none_or_int, default=None,
142	help="public mode - retrieves the public datasets")
143
144	# some arguments that override the default in the CONTROL file
145	parser.add_argument("--basetime", dest="basetime",
146	type=none_or_int, default=None,
147	help="base such as 00 or 12 (for half day retrievals)")
148	parser.add_argument("--step", dest="step",
149	type=none_or_str, default=None,
150	help="steps such as 00/to/48")
151	parser.add_argument("--levelist", dest="levelist",
152	type=none_or_str, default=None,
153	help="Vertical levels to be retrieved, e.g. 30/to/60")
154	parser.add_argument("--area", dest="area",
155	type=none_or_str, default=None,
156	help="area defined as north/west/south/east")
157
158	# set the working directories
159	parser.add_argument("--inputdir", dest="inputdir",
160	type=none_or_str, default=None,
161	help="root directory for storing intermediate files")
162	parser.add_argument("--outputdir", dest="outputdir",
163	type=none_or_str, default=None,
164	help="root directory for storing output files")
165	parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
166	type=none_or_str, default=None,
167	help="FLEXPART root directory (to find grib2flexpart \
168	and COMMAND file)\n Normally flex_extract resides in \
169	the scripts directory of the FLEXPART distribution")
170
171	# this is only used by prepare_flexpart.py to rerun a postprocessing step
172	parser.add_argument("--ppid", dest="ppid",
173	type=none_or_int, default=None,
174	help="specify parent process id for \
175	rerun of prepare_flexpart")
176
177	# arguments for job submission to ECMWF, only needed by submit.py
178	parser.add_argument("--job_template", dest='job_template',
179	type=none_or_str, default="job.temp",
180	help="job template file for submission to ECMWF")
181	parser.add_argument("--queue", dest="queue",
182	type=none_or_str, default=None,
183	help="queue for submission to ECMWF \
184	(e.g. ecgate or cca )")
185
186	args = parser.parse_args()
187
188	return args
189
190	def read_ecenv(filename):
191	'''Reads the file into a dictionary where the key values are the parameter
192	names.
193
194	Parameters
195	----------
196	filename : :obj:`string`
197	Path to file where the ECMWF environment parameters are stored.
198
199	Return
200	------
201	envs : :obj:`dictionary`
202	Contains the environment parameter ecuid, ecgid, gateway
203	and destination for ECMWF server environments.
204	'''
205	envs= {}
206
207	with open(filename, 'r') as f:
208	for line in f:
209	data = line.strip().split()
210	envs[str(data[0])] = str(data[1])
211
212	return envs
213
214	def clean_up(c):
215	'''Remove all files from intermediate directory (inputdir).
216
217	Parameters
218	----------
219	c : :obj:`ControlFile`
220	Contains all the parameters of CONTROL file and
221	command line.
222
223	Return
224	------
225
226	'''
227
228	print("clean_up")
229
230	cleanlist = glob.glob(c.inputdir + "/*")
231	for clist in cleanlist:
232	if c.prefix not in clist:
233	silent_remove(clist)
234	if c.ecapi is False and (c.ectrans == '1' or c.ecstorage == '1'):
235	silent_remove(clist)
236
237	print("Done")
238
239	return
240
241
242	def my_error(users, message='ERROR'):
243	'''Prints a specified error message which can be passed to the function
244	before exiting the program.
245
246	Parameters
247	----------
248	user : :obj:`list` of :obj:`string`
249	Contains all email addresses which should be notified.
250	It might also contain just the ecmwf user name which wil trigger
251	mailing to the associated email address for this user.
252
253	message : :obj:`string`, optional
254	Error message. Default value is "ERROR".
255
256	Return
257	------
258
259	'''
260
261	print(message)
262
263	# comment if user does not want email notification directly from python
264	for user in users:
265	if '${USER}' in user:
266	user = os.getenv('USER')
267	try:
268	p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ERROR',
269	os.path.expandvars(user)],
270	stdin=subprocess.PIPE,
271	stdout=subprocess.PIPE,
272	stderr=subprocess.PIPE,
273	bufsize=1)
274	trace = '\n'.join(traceback.format_stack())
275	pout = p.communicate(input=message + '\n\n' + trace)[0]
276	except ValueError as e:
277	print('ERROR: ', e)
278	sys.exit('Email could not be sent!')
279	else:
280	print('Email sent to ' + os.path.expandvars(user) + ' ' +
281	pout.decode())
282
283	sys.exit(1)
284
285	return
286
287
288	def normal_exit(users, message='Done!'):
289	'''Prints a specific exit message which can be passed to the function.
290
291	Parameters
292	----------
293	user : :obj:`list` of :obj:`string`
294	Contains all email addresses which should be notified.
295	It might also contain just the ecmwf user name which wil trigger
296	mailing to the associated email address for this user.
297
298	message : :obj:`string`, optional
299	Message for exiting program. Default value is "Done!".
300
301	Return
302	------
303
304	'''
305	print(message)
306
307	# comment if user does not want notification directly from python
308	for user in users:
309	if '${USER}' in user:
310	user = os.getenv('USER')
311	try:
312	p = subprocess.Popen(['mail', '-s flex_extract_v7.1 normal exit',
313	os.path.expandvars(user)],
314	stdin=subprocess.PIPE,
315	stdout=subprocess.PIPE,
316	stderr=subprocess.PIPE,
317	bufsize=1)
318	pout = p.communicate(input=message+'\n\n')[0]
319	except ValueError as e:
320	print('ERROR: ', e)
321	print('Email could not be sent!')
322	else:
323	print('Email sent to ' + os.path.expandvars(user) + ' ' +
324	pout.decode())
325
326	return
327
328
329	def product(args, *kwds):
330	'''This method combines the single characters of the passed arguments
331	with each other. So that each character of each argument value
332	will be combined with each character of the other arguments as a tuple.
333
334	Note
335	----
336	This method is taken from an example at the ECMWF wiki website.
337	https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
338
339	Example
340	-------
341	product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
342
343	product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
344
345	Parameters
346	----------
347	\*args : :obj:`tuple`
348	Positional arguments (arbitrary number).
349
350	\\kwds : :obj:`dictionary`
351	Contains all the keyword arguments from \*args.
352
353	Return
354	------
355	prod : :obj:`tuple`
356	Return will be done with "yield". A tuple of combined arguments.
357	See example in description above.
358	'''
359	pools = map(tuple, args) * kwds.get('repeat', 1)
360	result = [[]]
361	for pool in pools:
362	result = [x + [y] for x in result for y in pool]
363	for prod in result:
364	yield tuple(prod)
365
366	return
367
368
369	def silent_remove(filename):
370	'''Remove file if it exists.
371	The function does not fail if the file does not exist.
372
373	Parameters
374	----------
375	filename : :obj:`string`
376	The name of the file to be removed without notification.
377
378	Return
379	------
380
381	'''
382	try:
383	os.remove(filename)
384	except OSError as e:
385	if e.errno != errno.ENOENT:
386	# errno.ENOENT = no such file or directory
387	raise # re-raise exception if a different error occured
388
389	return
390
391
392	def init128(filepath):
393	'''Opens and reads the grib file with table 128 information.
394
395	Parameters
396	----------
397	filepath : :obj:`string`
398	Path to file of ECMWF grib table number 128.
399
400	Return
401	------
402	table128 : :obj:`dictionary`
403	Contains the ECMWF grib table 128 information.
404	The key is the parameter number and the value is the
405	short name of the parameter.
406	'''
407	table128 = dict()
408	with open(filepath) as f:
409	fdata = f.read().split('\n')
410	for data in fdata:
411	if data[0] != '!':
412	table128[data[0:3]] = data[59:64].strip()
413
414	return table128
415
416
417	def to_param_id(pars, table):
418	'''Transform parameter names to parameter ids with ECMWF grib table 128.
419
420	Parameters
421	----------
422	pars : :obj:`string`
423	Addpar argument from CONTROL file in the format of
424	parameter names instead of ids. The parameter short
425	names are sepearted with "/" and they are passed as
426	one single string.
427
428	table : :obj:`dictionary`
429	Contains the ECMWF grib table 128 information.
430	The key is the parameter number and the value is the
431	short name of the parameter.
432
433	Return
434	------
435	ipar : :obj:`list` of :obj:`integer`
436	List of addpar parameters from CONTROL file transformed to
437	parameter ids in the format of integer.
438	'''
439	cpar = pars.upper().split('/')
440	ipar = []
441	for par in cpar:
442	for k, v in table.iteritems():
443	if par == k or par == v:
444	ipar.append(int(k))
445	break
446	else:
447	print('Warning: par ' + par + ' not found in table 128')
448
449	return ipar
450
451	def get_list_as_string(list_obj, concatenate_sign=', '):
452	'''Converts a list of arbitrary content into a single string.
453
454	Parameters
455	----------
456	list_obj : :obj:`list`
457	A list with arbitrary content.
458
459	concatenate_sign : :obj:`string`, optional
460	A string which is used to concatenate the single
461	list elements. Default value is ", ".
462
463	Return
464	------
465	str_of_list : :obj:`string`
466	The content of the list as a single string.
467	'''
468
469	str_of_list = concatenate_sign.join(str(l) for l in list_obj)
470
471	return str_of_list
472
473	def make_dir(directory):
474	'''Creates a directory and gives a warning if the directory
475	already exists. The program stops only if there is another problem.
476
477	Parameters
478	----------
479	directory : :obj:`string`
480	The directory name including the path which should be created.
481
482	Return
483	------
484
485	'''
486	try:
487	os.makedirs(directory)
488	except OSError as e:
489	if e.errno != errno.EEXIST:
490	# errno.EEXIST = directory already exists
491	raise # re-raise exception if a different error occured
492	else:
493	print('WARNING: Directory {0} already exists!'.format(directory))
494
495	return
496
497	def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
498	'''Uses the ecaccess-file-put command to send a file to the ECMWF servers.
499
500	Note
501	----
502	The return value is just for testing reasons. It does not have
503	to be used from the calling function since the whole error handling
504	is done in here.
505
506	Parameters
507	----------
508	ecd : :obj:`string`
509	The path were the file is stored.
510
511	filename : :obj:`string`
512	The name of the file to send to the ECMWF server.
513
514	target : :obj:`string`
515	The target queue where the file should be sent to.
516
517	ecuid : :obj:`string`
518	The user id on ECMWF server.
519
520	ecgid : :obj:`string`
521	The group id on ECMWF server.
522
523	Return
524	------
525	rcode : :obj:`string`
526	Resulting code of command execution. If successful the string
527	will be empty.
528	'''
529
530	try:
531	rcode = subprocess.check_output(['ecaccess-file-put',
532	ecd + '/' + filename,
533	target + ':/home/ms/' +
534	ecgid + '/' + ecuid +
535	'/' + filename],
536	stderr=subprocess.STDOUT)
537	except subprocess.CalledProcessError as e:
538	print('... ERROR CODE:\n ... ' + str(e.returncode))
539	print('... ERROR MESSAGE:\n ... ' + str(e))
540
541	print('\n... Do you have a valid ecaccess certification key?')
542	sys.exit('... ECACCESS-FILE-PUT FAILED!')
543
544	return rcode
545
546	def submit_job_to_ecserver(target, jobname):
547	'''Uses ecaccess-job-submit command to submit a job to the ECMWF server.
548
549	Note
550	----
551	The return value is just for testing reasons. It does not have
552	to be used from the calling function since the whole error handling
553	is done in here.
554
555	Parameters
556	----------
557	target : :obj:`string`
558	The target where the file should be sent to, e.g. the queue.
559
560	jobname : :obj:`string`
561	The name of the jobfile to be submitted to the ECMWF server.
562
563	Return
564	------
565	rcode : :obj:`string`
566	Resulting code of command execution. If successful the string
567	will contain an integer number, representing the id of the job
568	at the ecmwf server.
569	'''
570
571	try:
572	rcode = subprocess.check_output(['ecaccess-job-submit',
573	'-queueName', target,
574	jobname])
575	except subprocess.CalledProcessError as e:
576	print('... ERROR CODE:\n ... ' + str(e.returncode))
577	print('... ERROR MESSAGE:\n ... ' + str(e))
578
579
580	print('\n... Do you have a valid ecaccess certification key?')
581	sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
582
583	return rcode

Note: See TracBrowser for help on using the repository browser.

Download in other formats: