Context Navigation

source: flex_extract.git/source/python/mods/tools.py @ 786cfd6

ctbtodev

Last change on this file since 786cfd6 was 786cfd6, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
corrected type of ppid to str
Property mode set to `100644`
File size: 19.5 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	#*******************************************************************************
4	# @Author: Anne Philipp (University of Vienna)
5	#
6	# @Date: May 2018
7	#
8	# @Change History:
9	# October 2014 - Anne Fouilloux (University of Oslo)
10	# - created functions silent_remove and product (taken from ECMWF)
11	#
12	# November 2015 - Leopold Haimberger (University of Vienna)
13	# - created functions: interpret_args_and_control, clean_up
14	# my_error, normal_exit, init128, to_param_id
15	#
16	# April 2018 - Anne Philipp (University of Vienna):
17	# - applied PEP8 style guide
18	# - added documentation
19	# - moved all functions from file Flexparttools to this file tools
20	# - added function get_list_as_string
21	# - seperated args and control interpretation
22	#
23	# @License:
24	# (C) Copyright 2014-2018.
25	#
26	# This software is licensed under the terms of the Apache Licence Version 2.0
27	# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
28	#
29	# @Modul Description:
30	# This module contains a couple of helpful functions which are
31	# used in different places in flex_extract.
32	#
33	# @Module Content:
34	# - get_cmdline_arguments
35	# - clean_up
36	# - my_error
37	# - normal_exit
38	# - product
39	# - silent_remove
40	# - init128
41	# - to_param_id
42	# - get_list_as_string
43	# - make_dir
44	#
45	#*******************************************************************************
46
47	# ------------------------------------------------------------------------------
48	# MODULES
49	# ------------------------------------------------------------------------------
50	import os
51	import errno
52	import sys
53	import glob
54	import subprocess
55	import traceback
56	import exceptions
57	from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
58
59	# ------------------------------------------------------------------------------
60	# FUNCTIONS
61	# ------------------------------------------------------------------------------
62
63	def none_or_str(value):
64	'''Converts the input string into pythons None-type if the string
65	contains string "None".
66
67	Parameters
68	----------
69	value : :obj:`string`
70	String to be checked for the "None" word.
71
72	Return
73	------
74	None or value:
75	Return depends on the content of the input value. If it was "None",
76	then the python type None is returned. Otherwise the string itself.
77	'''
78	if value == 'None':
79	return None
80	return value
81
82	def none_or_int(value):
83	'''Converts the input string into pythons None-type if the string
84	contains string "None". Otherwise it is converted to an integer value.
85
86	Parameters
87	----------
88	value : :obj:`string`
89	String to be checked for the "None" word.
90
91	Return
92	------
93	None or int(value):
94	Return depends on the content of the input value. If it was "None",
95	then the python type None is returned. Otherwise the string is
96	converted into an integer value.
97	'''
98	if value == 'None':
99	return None
100	return int(value)
101
102	def check_filepattern(filename):
103	'''
104	'''
105	if '.ksh' in filename:
106	return True
107	return False
108
109
110
111	def get_cmdline_arguments():
112	'''Decomposes the command line arguments and assigns them to variables.
113	Apply default values for non mentioned arguments.
114
115	Parameters
116	----------
117
118	Return
119	------
120	args : :obj:`Namespace`
121	Contains the commandline arguments from script/program call.
122	'''
123
124	parser = ArgumentParser(description='Retrieve FLEXPART input from \
125	ECMWF MARS archive',
126	formatter_class=ArgumentDefaultsHelpFormatter)
127
128	# the most important arguments
129	parser.add_argument("--start_date", dest="start_date",
130	type=none_or_str, default=None,
131	help="start date YYYYMMDD")
132	parser.add_argument("--end_date", dest="end_date",
133	type=none_or_str, default=None,
134	help="end_date YYYYMMDD")
135	parser.add_argument("--date_chunk", dest="date_chunk",
136	type=none_or_int, default=None,
137	help="# of days to be retrieved at once")
138	parser.add_argument("--controlfile", dest="controlfile",
139	type=none_or_str, default='CONTROL.temp',
140	help="file with CONTROL parameters")
141
142	# parameter for extra output information
143	parser.add_argument("--debug", dest="debug",
144	type=none_or_int, default=None,
145	help="debug mode - leave temporary files intact")
146	parser.add_argument("--request", dest="request",
147	type=none_or_int, default=None,
148	help="list all mars request in file mars_requests.dat \
149	and skip submission to mars")
150	parser.add_argument("--public", dest="public",
151	type=none_or_int, default=None,
152	help="public mode - retrieves the public datasets")
153
154	# some arguments that override the default in the CONTROL file
155	parser.add_argument("--basetime", dest="basetime",
156	type=none_or_int, default=None,
157	help="base such as 00 or 12 (for half day retrievals)")
158	parser.add_argument("--step", dest="step",
159	type=none_or_str, default=None,
160	help="steps such as 00/to/48")
161	parser.add_argument("--levelist", dest="levelist",
162	type=none_or_str, default=None,
163	help="Vertical levels to be retrieved, e.g. 30/to/60")
164	parser.add_argument("--area", dest="area",
165	type=none_or_str, default=None,
166	help="area defined as north/west/south/east")
167
168	# set the working directories
169	parser.add_argument("--inputdir", dest="inputdir",
170	type=none_or_str, default=None,
171	help="root directory for storing intermediate files")
172	parser.add_argument("--outputdir", dest="outputdir",
173	type=none_or_str, default=None,
174	help="root directory for storing output files")
175	parser.add_argument("--flexpart_root_scripts", dest="flexpart_root_scripts",
176	type=none_or_str, default=None,
177	help="FLEXPART root directory (to find grib2flexpart \
178	and COMMAND file)\n Normally flex_extract resides in \
179	the scripts directory of the FLEXPART distribution")
180
181	# this is only used by prepare_flexpart.py to rerun a postprocessing step
182	parser.add_argument("--ppid", dest="ppid",
183	type=none_or_str, default=None,
184	help="specify parent process id for \
185	rerun of prepare_flexpart")
186
187	# arguments for job submission to ECMWF, only needed by submit.py
188	parser.add_argument("--job_template", dest='job_template',
189	type=none_or_str, default="job.temp",
190	help="job template file for submission to ECMWF")
191	parser.add_argument("--queue", dest="queue",
192	type=none_or_str, default=None,
193	help="queue for submission to ECMWF \
194	(e.g. ecgate or cca )")
195
196	args = parser.parse_args()
197
198	return args
199
200	def read_ecenv(filepath):
201	'''Reads the file into a dictionary where the key values are the parameter
202	names.
203
204	Parameters
205	----------
206	filepath : :obj:`string`
207	Path to file where the ECMWF environment parameters are stored.
208
209	Return
210	------
211	envs : :obj:`dictionary`
212	Contains the environment parameter ecuid, ecgid, gateway
213	and destination for ECMWF server environments.
214	'''
215	envs= {}
216	try:
217	with open(filepath, 'r') as f:
218	for line in f:
219	data = line.strip().split()
220	envs[str(data[0])] = str(data[1])
221	except OSError as e:
222	print('... ERROR CODE: ' + str(e.errno))
223	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
224
225	sys.exit('\n... Error occured while trying to read ECMWF_ENV '
226	'file: ' + str(filepath))
227
228	return envs
229
230	def clean_up(c):
231	'''Remove files from the intermediate directory (inputdir).
232
233	It keeps the final FLEXPART input files if program runs without
234	ECMWF Api and keywords "ectrans" or "ecstorage" are set to "1".
235
236	Parameters
237	----------
238	c : :obj:`ControlFile`
239	Contains all the parameters of CONTROL file and
240	command line.
241
242	Return
243	------
244
245	'''
246
247	print("... clean inputdir!")
248
249	cleanlist = glob.glob(os.path.join(c.inputdir, "*"))
250
251	if cleanlist:
252	for element in cleanlist:
253	if c.prefix not in element:
254	silent_remove(element)
255	if c.ecapi is False and (c.ectrans == 1 or c.ecstorage == 1):
256	silent_remove(element)
257	print("... done!")
258	else:
259	print("... nothing to clean!")
260
261	return
262
263
264	def my_error(users, message='ERROR'):
265	'''Prints a specified error message which can be passed to the function
266	before exiting the program.
267
268	Parameters
269	----------
270	user : :obj:`list` of :obj:`string`
271	Contains all email addresses which should be notified.
272	It might also contain just the ecmwf user name which wil trigger
273	mailing to the associated email address for this user.
274
275	message : :obj:`string`, optional
276	Error message. Default value is "ERROR".
277
278	Return
279	------
280
281	'''
282
283	trace = '\n'.join(traceback.format_stack())
284	full_message = message + '\n\n' + trace
285
286	print(full_message)
287
288	send_mail(users, 'ERROR', full_message)
289
290	sys.exit(1)
291
292	return
293
294
295	def send_mail(users, success_mode, message):
296	'''Prints a specific exit message which can be passed to the function.
297
298	Parameters
299	----------
300	users : :obj:`list` of :obj:`string`
301	Contains all email addresses which should be notified.
302	It might also contain just the ecmwf user name which wil trigger
303	mailing to the associated email address for this user.
304
305	success_mode : :obj:``string`
306	States the exit mode of the program to put into
307	the mail subject line.
308
309	message : :obj:`string`, optional
310	Message for exiting program. Default value is "Done!".
311
312	Return
313	------
314
315	'''
316
317	for user in users:
318	if '${USER}' in user:
319	user = os.getenv('USER')
320	try:
321	p = subprocess.Popen(['mail', '-s flex_extract_v7.1 ' +
322	success_mode, os.path.expandvars(user)],
323	stdin=subprocess.PIPE,
324	stdout=subprocess.PIPE,
325	stderr=subprocess.PIPE,
326	bufsize=1)
327	pout = p.communicate(input=message + '\n\n')[0]
328	except ValueError as e:
329	print('... ERROR: ' + str(e))
330	sys.exit('... Email could not be sent!')
331	except OSError as e:
332	print('... ERROR CODE: ' + str(e.errno))
333	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
334	sys.exit('... Email could not be sent!')
335	else:
336	print('Email sent to ' + os.path.expandvars(user))
337
338	return
339
340
341	def normal_exit(message='Done!'):
342	'''Prints a specific exit message which can be passed to the function.
343
344	Parameters
345	----------
346	message : :obj:`string`, optional
347	Message for exiting program. Default value is "Done!".
348
349	Return
350	------
351
352	'''
353
354	print(str(message))
355
356	return
357
358
359	def product(args, *kwds):
360	'''Creates combinations of all passed arguments.
361
362	This method combines the single characters of the passed arguments
363	with each other. So that each character of each argument value
364	will be combined with each character of the other arguments as a tuple.
365
366	Note
367	----
368	This method is taken from an example at the ECMWF wiki website.
369	https://software.ecmwf.int/wiki/display/GRIB/index.py; 2018-03-16
370
371	Example
372	-------
373	product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
374
375	product(range(2), repeat = 3) --> 000 001 010 011 100 101 110 111
376
377	Parameters
378	----------
379	\*args : :obj:`list` or :obj:`string`
380	Positional arguments (arbitrary number).
381
382	\\kwds : :obj:`dictionary`
383	Contains all the keyword arguments from \*args.
384
385	Return
386	------
387	prod : :obj:`tuple`
388	Return will be done with "yield". A tuple of combined arguments.
389	See example in description above.
390	'''
391	try:
392	pools = map(tuple, args) * kwds.get('repeat', 1)
393	result = [[]]
394	for pool in pools:
395	result = [x + [y] for x in result for y in pool]
396	for prod in result:
397	yield tuple(prod)
398	except TypeError as e:
399	sys.exit('... PRODUCT GENERATION FAILED!')
400
401	return
402
403
404	def silent_remove(filename):
405	'''Remove file if it exists.
406	The function does not fail if the file does not exist.
407
408	Parameters
409	----------
410	filename : :obj:`string`
411	The name of the file to be removed without notification.
412
413	Return
414	------
415
416	'''
417	try:
418	os.remove(filename)
419	except OSError as e:
420	# errno.ENOENT = no such file or directory
421	if e.errno == errno.ENOENT:
422	pass
423	else:
424	raise # re-raise exception if a different error occured
425
426	return
427
428
429	def init128(filepath):
430	'''Opens and reads the grib file with table 128 information.
431
432	Parameters
433	----------
434	filepath : :obj:`string`
435	Path to file of ECMWF grib table number 128.
436
437	Return
438	------
439	table128 : :obj:`dictionary`
440	Contains the ECMWF grib table 128 information.
441	The key is the parameter number and the value is the
442	short name of the parameter.
443	'''
444	table128 = dict()
445	try:
446	with open(filepath) as f:
447	fdata = f.read().split('\n')
448	except OSError as e:
449	print('... ERROR CODE: ' + str(e.errno))
450	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
451
452	sys.exit('\n... Error occured while trying to read parameter '
453	'table file: ' + str(filepath))
454	else:
455	for data in fdata:
456	if data[0] != '!':
457	table128[data[0:3]] = data[59:64].strip()
458
459	return table128
460
461
462	def to_param_id(pars, table):
463	'''Transform parameter names to parameter ids with ECMWF grib table 128.
464
465	Parameters
466	----------
467	pars : :obj:`string`
468	Addpar argument from CONTROL file in the format of
469	parameter names instead of ids. The parameter short
470	names are sepearted with "/" and they are passed as
471	one single string.
472
473	table : :obj:`dictionary`
474	Contains the ECMWF grib table 128 information.
475	The key is the parameter number and the value is the
476	short name of the parameter.
477
478	Return
479	------
480	ipar : :obj:`list` of :obj:`integer`
481	List of addpar parameters from CONTROL file transformed to
482	parameter ids in the format of integer.
483	'''
484	if not pars:
485	return []
486	if not isinstance(pars, str):
487	pars=str(pars)
488
489	cpar = pars.upper().split('/')
490	ipar = []
491	for par in cpar:
492	for k, v in table.iteritems():
493	if par == k or par == v:
494	ipar.append(int(k))
495	break
496	else:
497	print('Warning: par ' + par + ' not found in table 128')
498
499	return ipar
500
501	def get_list_as_string(list_obj, concatenate_sign=', '):
502	'''Converts a list of arbitrary content into a single string.
503
504	Parameters
505	----------
506	list_obj : :obj:`list`
507	A list with arbitrary content.
508
509	concatenate_sign : :obj:`string`, optional
510	A string which is used to concatenate the single
511	list elements. Default value is ", ".
512
513	Return
514	------
515	str_of_list : :obj:`string`
516	The content of the list as a single string.
517	'''
518
519	if not isinstance(list_obj, list):
520	list_obj = list(list_obj)
521	str_of_list = concatenate_sign.join(str(l) for l in list_obj)
522
523	return str_of_list
524
525	def make_dir(directory):
526	'''Creates a directory.
527
528	It gives a warning if the directory already exists and skips process.
529	The program stops only if there is another problem.
530
531	Parameters
532	----------
533	directory : :obj:`string`
534	The path to directory which should be created.
535
536	Return
537	------
538
539	'''
540	try:
541	os.makedirs(directory)
542	except OSError as e:
543	# errno.EEXIST = directory already exists
544	if e.errno == errno.EEXIST:
545	print('WARNING: Directory {0} already exists!'.format(directory))
546	else:
547	raise # re-raise exception if a different error occured
548
549	return
550
551	def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
552	'''Uses the ecaccess-file-put command to send a file to the ECMWF servers.
553
554	Note
555	----
556	The return value is just for testing reasons. It does not have
557	to be used from the calling function since the whole error handling
558	is done in here.
559
560	Parameters
561	----------
562	ecd : :obj:`string`
563	The path were the file is stored.
564
565	filename : :obj:`string`
566	The name of the file to send to the ECMWF server.
567
568	target : :obj:`string`
569	The target queue where the file should be sent to.
570
571	ecuid : :obj:`string`
572	The user id on ECMWF server.
573
574	ecgid : :obj:`string`
575	The group id on ECMWF server.
576
577	Return
578	------
579
580	'''
581
582	try:
583	subprocess.check_output(['ecaccess-file-put',
584	ecd + '/' + filename,
585	target + ':/home/ms/' +
586	ecgid + '/' + ecuid +
587	'/' + filename],
588	stderr=subprocess.STDOUT)
589	except subprocess.CalledProcessError as e:
590	print('... ERROR CODE: ' + str(e.returncode))
591	print('... ERROR MESSAGE:\n \t ' + str(e))
592
593	print('\n... Do you have a valid ecaccess certification key?')
594	sys.exit('... ECACCESS-FILE-PUT FAILED!')
595	except OSError as e:
596	print('... ERROR CODE: ' + str(e.errno))
597	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
598
599	print('\n... Most likely the ECACCESS library is not available!')
600	sys.exit('... ECACCESS-FILE-PUT FAILED!')
601
602	return
603
604	def submit_job_to_ecserver(target, jobname):
605	'''Uses ecaccess-job-submit command to submit a job to the ECMWF server.
606
607	Note
608	----
609	The return value is just for testing reasons. It does not have
610	to be used from the calling function since the whole error handling
611	is done in here.
612
613	Parameters
614	----------
615	target : :obj:`string`
616	The target where the file should be sent to, e.g. the queue.
617
618	jobname : :obj:`string`
619	The name of the jobfile to be submitted to the ECMWF server.
620
621	Return
622	------
623	job_id : :obj:`int`
624	The id number of the job as a reference at the ecmwf server.
625	'''
626
627	try:
628	job_id = subprocess.check_output(['ecaccess-job-submit', '-queueName',
629	target, jobname])
630
631	except subprocess.CalledProcessError as e:
632	print('... ERROR CODE: ' + str(e.returncode))
633	print('... ERROR MESSAGE:\n \t ' + str(e))
634
635	print('\n... Do you have a valid ecaccess certification key?')
636	sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
637	except OSError as e:
638	print('... ERROR CODE: ' + str(e.errno))
639	print('... ERROR MESSAGE:\n \t ' + str(e.strerror))
640
641	print('\n... Most likely the ECACCESS library is not available!')
642	sys.exit('... ECACCESS-JOB-SUBMIT FAILED!')
643
644	return job_id

Note: See TracBrowser for help on using the repository browser.

Download in other formats: