[d69b677] | 1 | #!/usr/bin/env python |
---|
[efdb01a] | 2 | # -*- coding: utf-8 -*- |
---|
| 3 | #************************************************************************ |
---|
| 4 | # TODO AP |
---|
| 5 | # - Change History ist nicht angepasst ans File! |
---|
| 6 | # - add file description |
---|
| 7 | #************************************************************************ |
---|
| 8 | """ |
---|
| 9 | @Author: Anne Fouilloux (University of Oslo) |
---|
| 10 | |
---|
| 11 | @Date: October 2014 |
---|
| 12 | |
---|
| 13 | @ChangeHistory: |
---|
| 14 | November 2015 - Leopold Haimberger (University of Vienna): |
---|
| 15 | - using the WebAPI also for general MARS retrievals |
---|
| 16 | - job submission on ecgate and cca |
---|
| 17 | - job templates suitable for twice daily operational dissemination |
---|
| 18 | - dividing retrievals of longer periods into digestable chunks |
---|
| 19 | - retrieve also longer term forecasts, not only analyses and |
---|
| 20 | short term forecast data |
---|
| 21 | - conversion into GRIB2 |
---|
| 22 | - conversion into .fp format for faster execution of FLEXPART |
---|
| 23 | |
---|
| 24 | February 2018 - Anne Philipp (University of Vienna): |
---|
| 25 | - applied PEP8 style guide |
---|
| 26 | - added documentation |
---|
| 27 | - minor changes in programming style for consistence |
---|
| 28 | |
---|
| 29 | @License: |
---|
| 30 | (C) Copyright 2014-2018. |
---|
| 31 | |
---|
| 32 | This software is licensed under the terms of the Apache Licence Version 2.0 |
---|
| 33 | which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
---|
| 34 | |
---|
| 35 | @Requirements: |
---|
| 36 | - A standard python 2.6 or 2.7 installation |
---|
| 37 | - dateutils |
---|
| 38 | - ECMWF specific packages, all available from https://software.ecmwf.int/ |
---|
| 39 | ECMWF WebMARS, gribAPI with python enabled, emoslib and |
---|
| 40 | ecaccess web toolkit |
---|
| 41 | |
---|
| 42 | @Description: |
---|
| 43 | Further documentation may be obtained from www.flexpart.eu. |
---|
| 44 | |
---|
| 45 | """ |
---|
| 46 | # ------------------------------------------------------------------------------ |
---|
| 47 | # MODULES |
---|
| 48 | # ------------------------------------------------------------------------------ |
---|
[d69b677] | 49 | try: |
---|
| 50 | ecapi=True |
---|
| 51 | import ecmwfapi |
---|
| 52 | except ImportError: |
---|
| 53 | ecapi=False |
---|
| 54 | |
---|
| 55 | import calendar |
---|
| 56 | import shutil |
---|
| 57 | import datetime |
---|
| 58 | import time |
---|
[efdb01a] | 59 | import os |
---|
| 60 | import glob |
---|
| 61 | import sys |
---|
| 62 | import inspect |
---|
[d69b677] | 63 | # add path to submit.py to pythonpath so that python finds its buddies |
---|
| 64 | localpythonpath=os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) |
---|
| 65 | if localpythonpath not in sys.path: |
---|
| 66 | sys.path.append(localpythonpath) |
---|
| 67 | |
---|
[efdb01a] | 68 | from Control import Control |
---|
| 69 | from Tools import myerror, normalexit, \ |
---|
[64cf353] | 70 | interpret_args_and_control |
---|
[efdb01a] | 71 | from ECFlexpart import ECFlexpart |
---|
[64cf353] | 72 | |
---|
[efdb01a] | 73 | # ------------------------------------------------------------------------------ |
---|
| 74 | # FUNCTION |
---|
| 75 | # ------------------------------------------------------------------------------ |
---|
[64cf353] | 76 | def getMARSdata(args, c): |
---|
[d69b677] | 77 | |
---|
| 78 | |
---|
| 79 | if not os.path.exists(c.inputdir): |
---|
| 80 | os.makedirs(c.inputdir) |
---|
[64cf353] | 81 | print("start date %s " % (c.start_date)) |
---|
| 82 | print("end date %s " % (c.end_date)) |
---|
[d69b677] | 83 | |
---|
| 84 | if ecapi: |
---|
| 85 | server = ecmwfapi.ECMWFService("mars") |
---|
| 86 | else: |
---|
| 87 | server = False |
---|
| 88 | |
---|
[64cf353] | 89 | c.ecapi = ecapi |
---|
| 90 | print 'ecapi:', c.ecapi |
---|
[d69b677] | 91 | |
---|
[efdb01a] | 92 | # Retrieve EC data for running flexpart |
---|
[64cf353] | 93 | #AP change this variant to correct format conversion with datetime |
---|
| 94 | #AP import datetime and timedelta explicitly |
---|
| 95 | syear = int(c.start_date[:4]) |
---|
| 96 | smonth = int(c.start_date[4:6]) |
---|
| 97 | sday = int(c.start_date[6:]) |
---|
| 98 | start = datetime.date(year=syear, month=smonth, day=sday) |
---|
| 99 | startm1 = start - datetime.timedelta(days=1) |
---|
| 100 | if c.basetime == '00': |
---|
| 101 | start = startm1 |
---|
[efdb01a] | 102 | |
---|
[64cf353] | 103 | eyear = int(c.end_date[:4]) |
---|
| 104 | emonth = int(c.end_date[4:6]) |
---|
| 105 | eday = int(c.end_date[6:]) |
---|
| 106 | end = datetime.date(year=eyear, month=emonth, day=eday) |
---|
| 107 | if c.basetime == '00' or c.basetime == '12': |
---|
| 108 | endp1 = end + datetime.timedelta(days=1) |
---|
[d69b677] | 109 | else: |
---|
[64cf353] | 110 | endp1 = end + datetime.timedelta(days=2) |
---|
| 111 | |
---|
| 112 | datechunk = datetime.timedelta(days=int(c.date_chunk)) |
---|
[efdb01a] | 113 | |
---|
| 114 | # retrieving of accumulated data fields (flux data), (maximum one month) |
---|
| 115 | |
---|
| 116 | # remove old files |
---|
[64cf353] | 117 | print 'removing content of ' + c.inputdir |
---|
[efdb01a] | 118 | tobecleaned = glob.glob(c.inputdir + '/*_acc_*.' + \ |
---|
| 119 | str(os.getppid()) + '.*.grb') |
---|
[d69b677] | 120 | for f in tobecleaned: |
---|
| 121 | os.remove(f) |
---|
[64cf353] | 122 | |
---|
[efdb01a] | 123 | times = None |
---|
| 124 | # if forecast for maximum one day (upto 23h) are to be retrieved, |
---|
| 125 | # collect accumulation data (flux data) |
---|
| 126 | # with additional days in the beginning and at the end |
---|
| 127 | # (used for complete disaggregation of original period) |
---|
| 128 | if c.maxstep < 24: |
---|
| 129 | day = startm1 |
---|
| 130 | while day < endp1: |
---|
| 131 | # retrieve MARS data for the whole period |
---|
| 132 | flexpart = ECFlexpart(c, fluxes=True) |
---|
| 133 | tmpday = day + datechunk - datetime.timedelta(days=1) |
---|
| 134 | if tmpday < endp1: |
---|
| 135 | dates = day.strftime("%Y%m%d") + "/to/" + \ |
---|
| 136 | tmpday.strftime("%Y%m%d") |
---|
[d69b677] | 137 | else: |
---|
[efdb01a] | 138 | dates = day.strftime("%Y%m%d") + "/to/" + \ |
---|
| 139 | end.strftime("%Y%m%d") |
---|
[64cf353] | 140 | |
---|
[d69b677] | 141 | print "retrieve " + dates + " in dir " + c.inputdir |
---|
[efdb01a] | 142 | |
---|
[d69b677] | 143 | try: |
---|
| 144 | flexpart.retrieve(server, dates, times, c.inputdir) |
---|
| 145 | except IOError: |
---|
| 146 | myerror(c,'MARS request failed') |
---|
[64cf353] | 147 | |
---|
[efdb01a] | 148 | day += datechunk |
---|
| 149 | |
---|
| 150 | # if forecast data longer than 24h are to be retrieved, |
---|
| 151 | # collect accumulation data (flux data) |
---|
| 152 | # with the exact start and end date |
---|
| 153 | # (disaggregation will be done for the |
---|
| 154 | # exact time period with boundary conditions) |
---|
[d69b677] | 155 | else: |
---|
[efdb01a] | 156 | day = start |
---|
| 157 | while day <= end: |
---|
| 158 | # retrieve MARS data for the whole period |
---|
| 159 | flexpart = ECFlexpart(c, fluxes=True) |
---|
| 160 | tmpday = day + datechunk - datetime.timedelta(days=1) |
---|
| 161 | if tmpday < end: |
---|
| 162 | dates = day.strftime("%Y%m%d") + "/to/" + \ |
---|
| 163 | tmpday.trftime("%Y%m%d") |
---|
[d69b677] | 164 | else: |
---|
[efdb01a] | 165 | dates = day.strftime("%Y%m%d") + "/to/" + \ |
---|
| 166 | end.strftime("%Y%m%d") |
---|
[64cf353] | 167 | |
---|
[d69b677] | 168 | print "retrieve " + dates + " in dir " + c.inputdir |
---|
[64cf353] | 169 | |
---|
[efdb01a] | 170 | try: |
---|
| 171 | flexpart.retrieve(server, dates, times, c.inputdir) |
---|
| 172 | except IOError: |
---|
| 173 | myerror(c, 'MARS request failed') |
---|
| 174 | |
---|
| 175 | day += datechunk |
---|
[d69b677] | 176 | |
---|
[efdb01a] | 177 | # retrieving of normal data fields (non flux data), (maximum one month) |
---|
[d69b677] | 178 | |
---|
[efdb01a] | 179 | # remove old *__* files |
---|
| 180 | tobecleaned = glob.glob(c.inputdir + '/*__*.' + |
---|
| 181 | str(os.getppid()) + '.*.grb') |
---|
[d69b677] | 182 | for f in tobecleaned: |
---|
| 183 | os.remove(f) |
---|
[efdb01a] | 184 | day = start |
---|
| 185 | times = None |
---|
| 186 | while day <= end: |
---|
| 187 | # retrieve MARS data for the whole period |
---|
| 188 | flexpart = ECFlexpart(c, fluxes=False) |
---|
| 189 | tmpday = day+datechunk-datetime.timedelta(days=1) |
---|
| 190 | if tmpday < end: |
---|
| 191 | dates = day.strftime("%Y%m%d") + "/to/" + \ |
---|
| 192 | tmpday.strftime("%Y%m%d") |
---|
[d69b677] | 193 | else: |
---|
[efdb01a] | 194 | dates = day.strftime("%Y%m%d") + "/to/" + \ |
---|
| 195 | end.strftime("%Y%m%d") |
---|
| 196 | |
---|
[d69b677] | 197 | print "retrieve " + dates + " in dir " + c.inputdir |
---|
[64cf353] | 198 | |
---|
[efdb01a] | 199 | try: |
---|
| 200 | flexpart.retrieve(server, dates, times, c.inputdir) |
---|
| 201 | except IOError: |
---|
| 202 | myerror(c, 'MARS request failed') |
---|
| 203 | |
---|
| 204 | day += datechunk |
---|
[64cf353] | 205 | |
---|
[efdb01a] | 206 | return |
---|
[d69b677] | 207 | |
---|
| 208 | if __name__ == "__main__": |
---|
[64cf353] | 209 | |
---|
[efdb01a] | 210 | args, c = interpret_args_and_control() |
---|
| 211 | getMARSdata(args, c) |
---|
[d69b677] | 212 | normalexit(c) |
---|