Context Navigation

source: flex_extract.git/source/python/classes/ControlFile.py @ f2616a3

ctbtodev

Last change on this file since f2616a3 was f2616a3, checked in by Anne Philipp <anne.philipp@…>, 5 years ago
implemented a job split with a new parameter 'job_chunk' so that huge time periods can automatically be splitted
Property mode set to `100644`
File size: 13.9 KB

Rev	Line
[812283d]	1	#!/usr/bin/env python
	2	# -- coding: utf-8 --
	3	#*******************************************************************************
	4	# @Author: Leopold Haimberger (University of Vienna)
	5	#
	6	# @Date: November 2015
	7	#
	8	# @Change History:
	9	#
	10	# February 2018 - Anne Philipp (University of Vienna):
	11	# - applied PEP8 style guide
	12	# - added documentation
	13	# - applied some minor modifications in programming style/structure
	14	# - changed name of class Control to ControlFile for more
	15	# self-explanation naming
	16	# - outsource of class ControlFile
[54a8a01]	17	# - initialisation of class attributes ( to avoid high number of
	18	# conditional statements and set default values )
	19	# - divided assignment of attributes and the check of conditions
	20	# - outsourced the commandline argument assignments to control attributes
[812283d]	21	#
	22	# @License:
	23	# (C) Copyright 2015-2018.
	24	#
	25	# This software is licensed under the terms of the Apache Licence Version 2.0
	26	# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
	27	#
	28	# @Class Description:
	29	# The CONTROL file is the steering part of the FLEXPART extraction
	30	# software. All necessary parameters needed to retrieve the data fields
	31	# from the MARS archive for driving FLEXPART are set in a CONTROL file.
	32	# Some specific parameters like the start and end dates can be overwritten
	33	# by the command line parameters, but in generel all parameters needed
	34	# for a complete set of fields for FLEXPART can be set in the CONTROL file.
	35	#
	36	# @Class Content:
	37	# - __init__
[54a8a01]	38	# - __read_controlfile__
[812283d]	39	# - __str__
[54a8a01]	40	# - assign_args_to_control
	41	# - assign_envs_to_control
	42	# - check_conditions
	43	# - check_install_conditions
[ff99eae]	44	# - to_list
	45	#
	46	# @Class Attributes:
[54a8a01]	47	#
[812283d]	48	#
	49	#*******************************************************************************
	50
	51	# ------------------------------------------------------------------------------
	52	# MODULES
	53	# ------------------------------------------------------------------------------
	54	import os
[2fb99de]	55	import re
[54a8a01]	56	import sys
[812283d]	57	import inspect
[145fbe0]	58	import numpy as np
[ff99eae]	59
[ca867de]	60	# software specific classes and modules from flex_extract
	61	sys.path.append('../')
[2fb99de]	62	import _config
[70fee58]	63	from mods.tools import my_error, silent_remove
[145fbe0]	64	from mods.checks import (check_grid, check_area, check_levels, check_purefc,
	65	check_step, check_mail, check_queue, check_pathes,
	66	check_dates, check_maxstep, check_type, check_request,
	67	check_basetime, check_public, check_acctype,
	68	check_acctime, check_accmaxstep, check_time,
	69	check_logicals_type, check_len_type_time_step,
[f2616a3]	70	check_addpar, check_job_chunk)
[2fb99de]	71
[812283d]	72	# ------------------------------------------------------------------------------
	73	# CLASS
	74	# ------------------------------------------------------------------------------
[ff99eae]	75	class ControlFile(object):
[812283d]	76	'''
[708c667]	77	Contains the information which are stored in the CONTROL files.
[812283d]	78	'''
	79
	80	def __init__(self, filename):
[274f9ef]	81	'''Initialises the instance of ControlFile class and defines
	82	all class attributes with default values. Afterwards calls
	83	function __read_controlfile__ to read parameter from Control file.
[812283d]	84
[274f9ef]	85	Parameters
	86	----------
	87	filename : :obj:`string`
	88	Name of CONTROL file.
[812283d]	89
[274f9ef]	90	Return
	91	------
[812283d]	92
	93	'''
[54a8a01]	94
	95	# list of all possible class attributes and their default values
	96	self.controlfile = filename
	97	self.start_date = None
	98	self.end_date = None
	99	self.date_chunk = 3
	100	self.dtime = None
	101	self.basetime = None
	102	self.maxstep = None
	103	self.type = None
	104	self.time = None
	105	self.step = None
[0e576fc]	106	self.acctype = None
	107	self.acctime = None
	108	self.accmaxstep = None
[54a8a01]	109	self.marsclass = None
[5bad6ec]	110	self.dataset = None
[54a8a01]	111	self.stream = None
	112	self.number = 'OFF'
[2fb99de]	113	self.expver = '1'
[0e576fc]	114	self.gaussian = ''
[54a8a01]	115	self.grid = None
	116	self.area = ''
	117	self.left = None
	118	self.lower = None
	119	self.upper = None
	120	self.right = None
	121	self.level = None
	122	self.levelist = None
	123	self.resol = None
	124	self.gauss = 0
	125	self.accuracy = 24
	126	self.omega = 0
	127	self.omegadiff = 0
	128	self.eta = 0
	129	self.etadiff = 0
	130	self.etapar = 77
	131	self.dpdeta = 1
	132	self.smooth = 0
	133	self.format = 'GRIB1'
	134	self.addpar = None
	135	self.prefix = 'EN'
	136	self.cwc = 0
	137	self.wrf = 0
	138	self.ecfsdir = 'ectmp:/${USER}/econdemand/'
	139	self.mailfail = ['${USER}']
	140	self.mailops = ['${USER}']
	141	self.grib2flexpart = 0
	142	self.ecstorage = 0
	143	self.ectrans = 0
[25b14be]	144	self.inputdir = _config.PATH_INPUT_DIR
[ca867de]	145	self.outputdir = None
[145fbe0]	146	self.flexextractdir = _config.PATH_FLEXEXTRACT_DIR
[2fb99de]	147	self.exedir = _config.PATH_FORTRAN_SRC
[145fbe0]	148	self.flexpartdir = None
[2fb99de]	149	self.makefile = 'Makefile.gfortran'
[54a8a01]	150	self.destination = None
	151	self.gateway = None
	152	self.ecuid = None
	153	self.ecgid = None
	154	self.install_target = None
	155	self.debug = 0
[2fb99de]	156	self.request = 0
[5bad6ec]	157	self.public = 0
[96e1533]	158	self.ecapi = None
[4d3b052]	159	self.purefc = 0
[c274d9a]	160	self.rrint = 0
[2fb99de]	161
	162	self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff',
	163	'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage',
[4d3b052]	164	'ectrans', 'debug', 'request', 'public', 'purefc',
	165	'rrint']
[54a8a01]	166
[5c0a578]	167	self._read_controlfile()
[54a8a01]	168
	169	return
	170
[5c0a578]	171	def _read_controlfile(self):
[274f9ef]	172	'''Read CONTROL file and assign all CONTROL file variables.
[54a8a01]	173
[274f9ef]	174	Parameters
	175	----------
	176
	177	Return
	178	------
[54a8a01]	179
	180	'''
[4971f63]	181
	182	try:
[70fee58]	183	cfile = os.path.join(_config.PATH_CONTROLFILES, self.controlfile)
	184	with open(cfile) as f:
[4971f63]	185	fdata = f.read().split('\n')
	186	except IOError:
[70fee58]	187	print('Could not read CONTROL file "' + cfile + '"')
[4971f63]	188	print('Either it does not exist or its syntax is wrong.')
	189	print('Try "' + sys.argv[0].split('/')[-1] + \
	190	' -h" to print usage information')
	191	sys.exit(1)
[812283d]	192
	193	# go through every line and store parameter
	194	for ldata in fdata:
[87ae9a3]	195	if ldata and ldata[0] == '#':
	196	# ignore comment line in control file
	197	continue
	198	if '#' in ldata:
	199	# cut off comment
	200	ldata = ldata.split('#')[0]
[812283d]	201	data = ldata.split()
	202	if len(data) > 1:
	203	if 'm_' in data[0].lower():
	204	data[0] = data[0][2:]
	205	if data[0].lower() == 'class':
	206	data[0] = 'marsclass'
	207	if data[0].lower() == 'day1':
	208	data[0] = 'start_date'
	209	if data[0].lower() == 'day2':
	210	data[0] = 'end_date'
	211	if len(data) == 2:
	212	if '$' in data[1]:
	213	setattr(self, data[0].lower(), data[1])
	214	while '$' in data[1]:
	215	i = data[1].index('$')
	216	j = data[1].find('{')
	217	k = data[1].find('}')
	218	var = os.getenv(data[1][j+1:k])
	219	if var is not None:
	220	data[1] = data[1][:i] + var + data[1][k+1:]
	221	else:
[54a8a01]	222	my_error(self.mailfail,
	223	'Could not find variable '
	224	+ data[1][j+1:k] + ' while reading ' +
	225	self.controlfile)
[812283d]	226	setattr(self, data[0].lower() + '_expanded', data[1])
	227	else:
	228	if data[1].lower() != 'none':
	229	setattr(self, data[0].lower(), data[1])
	230	else:
	231	setattr(self, data[0].lower(), None)
	232	elif len(data) > 2:
	233	setattr(self, data[0].lower(), (data[1:]))
	234	else:
	235	pass
	236
[54a8a01]	237	return
	238
	239	def __str__(self):
[274f9ef]	240	'''Prepares a string which have all the ControlFile class attributes
	241	with its associated values. Each attribute is printed in one line and
	242	in alphabetical order.
	243
	244	Example
	245	-------
	246	'age': 10
	247	'color': 'Spotted'
	248	'kids': 0
	249	'legs': 2
	250	'name': 'Dog'
	251	'smell': 'Alot'
	252
	253	Parameters
	254	----------
	255
	256	Return
	257	------
	258	string
	259	Single string of concatenated ControlFile class attributes
	260	with their values
[54a8a01]	261	'''
	262	import collections
	263
[5bad6ec]	264	attrs = vars(self).copy()
[54a8a01]	265	attrs = collections.OrderedDict(sorted(attrs.items()))
	266
	267	return '\n'.join("%s: %s" % item for item in attrs.items())
	268
	269	def assign_args_to_control(self, args):
[274f9ef]	270	'''Overwrites the existing ControlFile instance attributes with
	271	the command line arguments.
[54a8a01]	272
[274f9ef]	273	Parameters
	274	----------
	275	args : :obj:`Namespace`
	276	Contains the commandline arguments from script/program call.
[54a8a01]	277
[274f9ef]	278	Return
	279	------
[54a8a01]	280
	281	'''
	282
	283	# get dictionary of command line parameters and eliminate all
	284	# parameters which are None (were not specified)
	285	args_dict = vars(args)
	286	arguments = {k : args_dict[k] for k in args_dict
	287	if args_dict[k] != None}
	288
	289	# assign all passed command line arguments to ControlFile instance
	290	for k, v in arguments.iteritems():
	291	setattr(self, str(k), v)
	292
	293	return
	294
	295	def assign_envs_to_control(self, envs):
[274f9ef]	296	'''Assigns the ECMWF environment parameter.
[54a8a01]	297
[274f9ef]	298	Parameters
	299	----------
	300	envs : :obj:`dictionary` of :obj:`strings`
	301	Contains the ECMWF environment parameternames "ECUID", "ECGID",
	302	"DESTINATION" and "GATEWAY" with its corresponding values.
	303	They were read from the file "ECMWF_ENV".
	304
	305	Return
	306	------
[54a8a01]	307
	308	'''
	309
	310	for k, v in envs.iteritems():
	311	setattr(self, str(k).lower(), str(v))
	312
	313	return
	314
[2fb99de]	315	def check_conditions(self, queue):
[274f9ef]	316	'''Checks a couple of necessary attributes and conditions,
	317	such as if they exist and contain values.
	318	Otherwise set default values.
	319
	320	Parameters
	321	----------
	322	queue : :obj:`string`
	323	Name of the queue if submitted to the ECMWF servers.
	324	Used to check if ecuid, ecgid, gateway and destination
	325	are set correctly and are not empty.
	326
	327	Return
	328	------
	329
[54a8a01]	330	'''
[145fbe0]	331	check_logicals_type(self, self.logicals)
	332
	333	self.mailfail = check_mail(self.mailfail)
	334
	335	self.mailops = check_mail(self.mailops)
[54a8a01]	336
[145fbe0]	337	check_queue(queue, self.gateway, self.destination,
	338	self.ecuid, self.ecgid)
[54a8a01]	339
[145fbe0]	340	self.outputdir, self.flexpartdir = check_pathes(self.inputdir,
	341	self.outputdir, self.flexpartdir, self.flexextractdir)
	342
	343	self.start_date, self.end_date = check_dates(self.start_date,
	344	self.end_date)
	345
	346	check_basetime(self.basetime)
[ca867de]	347
[3f36e42]	348	self.levelist, self.level = check_levels(self.levelist, self.level)
	349
[145fbe0]	350	self.step = check_step(self.step, self.mailfail)
[812283d]	351
[145fbe0]	352	self.maxstep = check_maxstep(self.maxstep, self.step)
[812283d]	353
[145fbe0]	354	check_request(self.request,
	355	os.path.join(self.inputdir, _config.FILE_MARS_REQUESTS))
[5bad6ec]	356
[145fbe0]	357	check_public(self.public, self.dataset)
[0e576fc]	358
[145fbe0]	359	self.type = check_type(self.type, self.step)
[0e576fc]	360
[145fbe0]	361	self.time = check_time(self.time)
[0e576fc]	362
[145fbe0]	363	self.type, self.time, self.step = check_len_type_time_step(self.type,
	364	self.time,
	365	self.step,
	366	self.maxstep,
	367	self.purefc)
[0e576fc]	368
[145fbe0]	369	self.acctype = check_acctype(self.acctype, self.type)
[0e576fc]	370
[145fbe0]	371	self.acctime = check_acctime(self.acctime, self.acctype, self.purefc)
[0e576fc]	372
[145fbe0]	373	self.accmaxstep = check_accmaxstep(self.accmaxstep, self.acctype,
	374	self.purefc, self.maxstep)
[0e576fc]	375
[4d3b052]	376	self.purefc = check_purefc(self.type)
	377
[3f36e42]	378	self.grid = check_grid(self.grid)
[97f4f4c]	379
[3f36e42]	380	self.area = check_area(self.grid, self.area, self.upper, self.lower,
	381	self.left, self.right)
[97f4f4c]	382
[145fbe0]	383	self.addpar = check_addpar(self.addpar)
	384
[f2616a3]	385	self.job_chunk = check_job_chunk(self.job_chunk)
[2d56c04]	386
[812283d]	387	return
	388
[ff99eae]	389	def to_list(self):
[274f9ef]	390	'''Just generates a list of strings containing the attributes and
	391	assigned values except the attributes "_expanded", "exedir",
[145fbe0]	392	"flexextractdir" and "flexpartdir".
[274f9ef]	393
	394	Parameters
	395	----------
	396
	397	Return
	398	------
	399	l : :obj:`list`
	400	A sorted list of the all ControlFile class attributes with
	401	their values except the attributes "_expanded", "exedir",
[145fbe0]	402	"flexextractdir" and "flexpartdir".
[812283d]	403	'''
	404
[54a8a01]	405	import collections
	406
[5bad6ec]	407	attrs = collections.OrderedDict(sorted(vars(self).copy().items()))
[54a8a01]	408
[812283d]	409	l = list()
	410
	411	for item in attrs.items():
	412	if '_expanded' in item[0]:
	413	pass
	414	elif 'exedir' in item[0]:
	415	pass
[145fbe0]	416	elif 'flexpartdir' in item[0]:
[812283d]	417	pass
[145fbe0]	418	elif 'flexextractdir' in item[0]:
[812283d]	419	pass
	420	else:
[ff99eae]	421	if isinstance(item[1], list):
[812283d]	422	stot = ''
	423	for s in item[1]:
	424	stot += s + ' '
	425
[ff2a11c]	426	l.append("%s %s\n" % (item[0], stot))
[812283d]	427	else:
[ff2a11c]	428	l.append("%s %s\n" % item)
[812283d]	429
	430	return sorted(l)
[ff99eae]	431

Note: See TracBrowser for help on using the repository browser.

Download in other formats: