- Timestamp:
- Oct 5, 2018, 5:20:48 PM (5 years ago)
- Branches:
- master, ctbto, dev
- Children:
- ae88f7d
- Parents:
- ca867de
- Location:
- source/python
- Files:
-
- 6 edited
Legend:
- Unmodified
- Added
- Removed
-
source/python/classes/ControlFile.py
rca867de r5bad6ec 114 114 self.step = None 115 115 self.marsclass = None 116 self.dataset = None 116 117 self.stream = None 117 118 self.number = 'OFF' … … 159 160 self.debug = 0 160 161 self.request = 0 162 self.public = 0 161 163 162 164 self.logicals = ['gauss', 'omega', 'omegadiff', 'eta', 'etadiff', 163 165 'dpdeta', 'cwc', 'wrf', 'grib2flexpart', 'ecstorage', 164 'ectrans', 'debug', 'request' ]166 'ectrans', 'debug', 'request', 'public'] 165 167 166 168 self.__read_controlfile__() … … 266 268 import collections 267 269 268 attrs = vars(self) 270 attrs = vars(self).copy() 269 271 attrs = collections.OrderedDict(sorted(attrs.items())) 270 272 … … 454 456 setattr(self, var, int(getattr(self, var))) 455 457 458 if self.public and not self.dataset: 459 print('ERROR: ') 460 print('If public mars data wants to be retrieved, ' 461 'the "dataset"-parameter has to be set in the control file!') 462 sys.exit(1) 463 456 464 return 457 465 … … 524 532 import collections 525 533 526 attrs = collections.OrderedDict(sorted(vars(self). items()))534 attrs = collections.OrderedDict(sorted(vars(self).copy().items())) 527 535 528 536 l = list() -
source/python/classes/EcFlexpart.py
rca867de r5bad6ec 89 89 import _config 90 90 from GribTools import GribTools 91 from mods.tools import init128, to_param_id, silent_remove, product, my_error 91 from mods.tools import (init128, to_param_id, silent_remove, product, 92 my_error, make_dir) 92 93 from MarsRetrieval import MarsRetrieval 93 94 import mods.disaggregation as disaggregation … … 143 144 144 145 self.inputdir = c.inputdir 146 self.dataset = c.dataset 145 147 self.basetime = c.basetime 146 148 self.dtime = c.dtime … … 338 340 Contains all parameter which have to be set for creating the 339 341 Mars Retrievals. The parameter are: 340 marsclass, stream, type, levtype, levelist, resol, gaussian,341 accuracy, grid, target, area, date, time, number, step, expver,342 param342 marsclass, dataset, stream, type, levtype, levelist, resol, 343 gaussian, accuracy, grid, target, area, date, time, number, 344 step, expver, param 343 345 344 346 @Return: … … 349 351 350 352 MR = MarsRetrieval(self.server, 353 self.public, 351 354 marsclass=par_dict['marsclass'], 355 dataset=par_dict['dataset'], 352 356 stream=par_dict['stream'], 353 357 type=par_dict['type'], … … 439 443 440 444 441 def retrieve(self, server, dates, request, inputdir='.'):445 def retrieve(self, server, dates, public, request, inputdir='.'): 442 446 ''' 443 447 @Description: … … 477 481 self.dates = dates 478 482 self.server = server 483 self.public = public 479 484 self.inputdir = inputdir 480 485 oro = False … … 488 493 # therefore be set in each request seperately 489 494 retr_param_dict = {'marsclass':self.marsclass, 495 'dataset':self.dataset, 490 496 'stream':None, 491 497 'type':None, … … 1265 1271 # create Options dir if necessary 1266 1272 if not os.path.exists(pwd + '/Options'): 1267 os.makedirs(pwd+'/Options')1273 make_dir(pwd+'/Options') 1268 1274 1269 1275 # read template COMMAND file -
source/python/classes/MarsRetrieval.py
rca867de r5bad6ec 83 83 ''' 84 84 85 def __init__(self, server, marsclass="ei", type="", levtype="",86 lev elist="", repres="", date="", resol="", stream="",87 area="", time="", step="", expver="1", number="",88 accuracy="", grid="", gaussian="", target="",85 def __init__(self, server, public, marsclass="ei", dataset="", type="", 86 levtype="", levelist="", repres="", date="", resol="", 87 stream="", area="", time="", step="", expver="1", 88 number="", accuracy="", grid="", gaussian="", target="", 89 89 param=""): 90 90 ''' … … 106 106 It is needed for the pythonic access of ECMWF data. 107 107 108 public: integer 109 Decides which Web API version is used: 110 0: member-state users and full archive access 111 1: public access and limited access to the public server and 112 datasets. Needs the parameter dataset. 113 Default is "0" and for member-state users. 114 108 115 marsclass: string, optional 109 116 Characterisation of dataset. E.g. EI (ERA-Interim), 110 117 E4 (ERA40), OD (Operational archive), ea (ERA5). 111 118 Default is the ERA-Interim dataset "ei". 119 120 dataset: string, optional 121 For public datasets there is the specific naming and parameter 122 dataset which has to be used to characterize the type of 123 data. Usually there is less data available, either in times, 124 domain or parameter. 125 Default is an empty string. 112 126 113 127 type: string, optional … … 289 303 290 304 self.server = server 305 self.public = public 291 306 self.marsclass = marsclass 307 self.dataset = dataset 292 308 self.type = type 293 309 self.levtype = levtype … … 325 341 ''' 326 342 # Get all class attributes and their values as a dictionary 327 attrs = vars(self) 343 attrs = vars(self).copy() 328 344 329 345 # iterate through all attributes and print them 330 346 # with their corresponding values 331 347 for item in attrs.items(): 332 if item[0] in 'server':348 if item[0] in ['server', 'public']: 333 349 pass 334 350 else: … … 358 374 ''' 359 375 # Get all class attributes and their values as a dictionary 360 attrs = vars(self) 376 attrs = vars(self).copy() 361 377 362 378 # open a file to store all requests to … … 367 383 # with their corresponding values 368 384 for item in attrs.items(): 369 if item[0] in 'server':385 if item[0] in ['server', 'public']: 370 386 pass 371 387 else: … … 396 412 397 413 # Get all class attributes and their values as a dictionary 398 attrs = vars(self) 414 attrs = vars(self).copy() 399 415 del attrs['server'] 416 del attrs['public'] 400 417 401 418 # open a file to store all requests to … … 425 442 ''' 426 443 # Get all class attributes and their values as a dictionary 427 attrs = vars(self) 428 429 # convert the dictionary of attributes into a comma 430 # seperated list of attributes with their values 431 # needed for the retrieval call 432 s = 'ret' 433 for k, v in attrs.iteritems(): 434 if k in 'server': 435 continue 436 if k == 'marsclass': 437 k = 'class' 438 if v == '': 439 continue 440 if k.lower() == 'target': 441 target = v 444 attrs = vars(self).copy() 445 446 # eliminate unnecessary attributes from the dictionary attrs 447 del attrs['server'] 448 del attrs['public'] 449 450 # exchange parameter name for marsclass 451 mclass = attrs.get('marsclass') 452 del attrs['marsclass'] 453 attrs['class'] = mclass 454 455 # prepare target variable as needed for the Web API mode 456 # within the dictionary for full access 457 # as a single variable for public access 458 target = attrs.get('target') 459 if not int(self.public): 460 del attrs['target'] 461 print('target: ' + target) 462 463 # find all keys without a value and convert all other values to strings 464 empty_keys = [] 465 for key, value in attrs.itteritems(): 466 if value == '': 467 empty_keys.append(str(key)) 442 468 else: 443 s = s + ',' + k + '=' + str(v) 469 attrs[key] = str(value) 470 471 # delete all empty parameter from the dictionary 472 for key in empty_keys: 473 del attrs[key] 444 474 445 475 # MARS request via Python script 446 if self.server is not False:476 if self.server: 447 477 try: 448 self.server.execute(s, target) 478 if self.public: 479 print('RETRIEVE PUBLIC DATA!') 480 self.server.retrieve(attrs) 481 else: 482 print('EXECUTE NON-PUBLIC RETRIEVAL!') 483 self.server.execute(attrs, target) 449 484 except: 450 print('MARS Request failed, \ 451 have you already registered at apps.ecmwf.int?') 485 e = sys.exc_info()[0] 486 print("ERROR: ", e) 487 print('MARS Request failed!') 488 if not self.public and os.stat(target).st_size == 0: 489 print('MARS Request returned no data - please check request') 452 490 raise IOError 453 if os.stat(target).st_size == 0: 454 print('MARS Request returned no data - please check request') 491 elif self.public and os.stat(target).st_size == 0: 492 print('Public MARS Request returned no data - ' 493 'please check request') 494 raise IOError 495 else: 455 496 raise IOError 456 497 # MARS request via extra process in shell 457 498 else: 458 s += ',target = "' + target + '"' 459 p = subprocess.Popen(['mars'], stdin=subprocess.PIPE, 499 request_str = 'ret' 500 for key, value in attrs.iteritems(): 501 request_str = request_str + ',' + key + '=' + str(value) 502 request_str += ',target="' + target + '"' 503 p = subprocess.Popen(['mars'], 504 stdin=subprocess.PIPE, 460 505 stdout=subprocess.PIPE, 461 stderr=subprocess.PIPE, bufsize=1) 462 pout = p.communicate(input=s)[0] 506 stderr=subprocess.PIPE, 507 bufsize=1) 508 pout = p.communicate(input=request_str)[0] 463 509 print(pout.decode()) 464 510 … … 466 512 print('MARS Request failed - please check request') 467 513 raise IOError 468 469 if os.stat(target).st_size == 0: 514 elif os.stat(target).st_size == 0: 470 515 print('MARS Request returned no data - please check request') 471 516 raise IOError 517 else: 518 raise 472 519 473 520 return -
source/python/mods/get_mars_data.py
rca867de r5bad6ec 54 54 sys.path.append('../') 55 55 import _config 56 from tools import my_error, normal_exit, get_cmdline_arguments, read_ecenv 56 from tools import (my_error, normal_exit, get_cmdline_arguments, 57 read_ecenv, make_dir) 57 58 from classes.EcFlexpart import EcFlexpart 58 59 from classes.UioFiles import UioFiles … … 114 115 115 116 if not os.path.exists(c.inputdir): 116 os.makedirs(c.inputdir)117 make_dir(c.inputdir) 117 118 118 119 if c.request == 0 or c.request == 2: … … 125 126 126 127 if ecapi: 127 server = ecmwfapi.ECMWFService("mars") 128 if c.public: 129 server = ecmwfapi.ECMWFDataServer() 130 else: 131 server = ecmwfapi.ECMWFService("mars") 128 132 else: 129 133 server = False … … 254 258 255 259 try: 256 flexpart.retrieve(server, dates, c. request, c.inputdir)260 flexpart.retrieve(server, dates, c.public, c.request, c.inputdir) 257 261 except IOError: 258 262 my_error(c.mailfail, 'MARS request failed') -
source/python/mods/prepare_flexpart.py
rca867de r5bad6ec 63 63 from classes.UioFiles import UioFiles 64 64 from classes.ControlFile import ControlFile 65 from tools import clean_up, get_cmdline_arguments, read_ecenv 65 from tools import clean_up, get_cmdline_arguments, read_ecenv, make_dir 66 66 from classes.EcFlexpart import EcFlexpart 67 67 … … 153 153 # create output dir if necessary 154 154 if not os.path.exists(c.outputdir): 155 os.makedirs(c.outputdir)155 make_dir(c.outputdir) 156 156 157 157 # get all files with flux data to be deaccumulated -
source/python/mods/tools.py
rca867de r5bad6ec 139 139 help="list all mars request in file mars_requests.dat \ 140 140 and skip submission to mars") 141 parser.add_argument("--public", dest="public", 142 type=none_or_int, default=None, 143 help="public mode - retrieves the public datasets") 141 144 142 145 # some arguments that override the default in the CONTROL file
Note: See TracChangeset
for help on using the changeset viewer.