Changes between Version 4 and Version 5 of FpInputMetMkavail


Ignore:
Timestamp:
Jun 10, 2018, 5:16:34 PM (6 years ago)
Author:
pesei
Comment:

replace by v6

Legend:

Unmodified
Added
Removed
Modified

  • FpInputMetMkavail

    v4 v5  
    77
    88
    9 {{{
     9{{{#!python
    1010#!/usr/bin/env python
    1111
    … …  
    1313SYNOPSIS
    1414
    15     mk_AVAILABLE.py:  [-s,--startdate] [-e,--enddate] [-m,--model /ECWMF/ERAI/GFS] [-p, --path] [-a, --avpath] [-i, --interval] [-h] [-v,--verbose] [--version]
     15  mk_AVAILABLE.py: [-s,--startdate] [-e,--enddate] [-m,--model /ECWMF/ERAI/EI/E5/EA/GFS] [-p, --path] [-a, --avpath] [-i, --interval] [-h] [-v,--verbose] [--version]
    1616
    1717DESCRIPTION
    1818
    19     mk_AVAIL is a script to create AVAILABLE files for FLEXPART windfields.
    20 
    21     Example usage:
    22     %mk_AVAIL.py -s 200601 -e 200701 -m ECMWF -p . -h
    23 
    24     would create a file: AVAIALABLE that contained paths
    25     of all files available in the current directory between 200601 and 200701.
    26     The default start date is 197001 and the default end date is TODAY.
    27    
    28     adapted and simplified version / Petra Seibert
    29     v.004: add ERA-Interim with EI as model, proper sorting of dates if year 2000 included, option to define a path for AVAILABLE / Petra Seibert 2015-10-21
    30     v.005: check whether any files are found before opening AVAILABLE, if not issue warning and stop. / Petra Seibert 2015-12-07
    31    
    32 QUESTIONS
    33 
    34     JFB: <jfb@nilu.no>
    35     v.003: petra.seibert at boku.ac.at
    36     v.004: petra.seibert at boku.ac.at
    37     v.005: petra.seibert at boku.ac.at
     19  mk_AVAIL is a script to create AVAILABLE files for FLEXPART windfields.
     20
     21  Example usage:
     22  %mk_AVAIL.py -s 200601 -e 200701 -m ECMWF -p . -h
     23
     24  would create a file: AVAIALABLE that contained paths
     25  of all files available in the current directory between 200601 and 200701.
     26  The default start date is 197001 and the default end date is TODAY.
     27 
     28  adapted and simplified version / Petra Seibert
     29  v.004: add ERA-Interim with EI as model, proper sorting of dates if year 2000 included, option to define a path for AVAILABLE / Petra Seibert 2015-10-21
     30  v.005: check whether any files are found before opening AVAILABLE, if not issue warning and stop. / Petra Seibert 2015-12-07
     31  v.006 PS 2018-04-24:   add ERA5 as model option
     32 
     33QUESTIONS to
     34
     35  JFB: <jfb@nilu.no>
     36  version>=003: petra.seibert at boku.ac.at
    3837
    3938LICENSE
    4039
    41     This script follows creative commons usage.
     40  This script follows creative commons usage.
     41  Valid-License-Identifier: CC-BY-4.0
    4242
    4343VERSION
    4444
    45     $Id: 0.004 $
     45  $Id: 0.006 $
    4646
    4747"""
    … …  
    6060def main ():
    6161
    62     global options, args
     62  global options, args, models
     63  version = '$Id: 0.006 $' 
     64 
     65  WIND_dir = '/xnilu_wrk/flex_wrk/WIND_FIELDS'
     66 
     67  # dict of dicts with model information
     68  # timeindex: index in the filne name string where the time information starts
     69  print  MODEL.keys()
     70
     71  AVAIL_head = """DATE   TIME     FILENAME       SPECIFICATIONS\
     72  \nYYYYMMDD HHMISS\
     73  \n________ ______    __________________    __________________\n"""
     74  start_date = options.startdate
     75  sy = int(start_date[0:4])
     76  sm = int(start_date[4:6])
     77  end_date = options.enddate
     78  ey = int(end_date[0:4])
     79  em = int(end_date[4:6])
     80 
     81  #Get the directory information
     82  M = MODEL[options.model]
     83  prfx = M['prefix']
     84  t = options.path
     85  avpath = options.avpath
     86 
     87 
     88  #Loop through the files in the directories
     89 
     90  warned = False
     91  d = t #directory
     92 
     93  tind = M['timeindex'] #timestamp index
     94  searchstr = os.path.join(t,prfx)
     95  print 'searchstring:',searchstr
     96  files = glob.glob(searchstr + '*')
     97  if options.verbose: print len(files), ' wind-field files found'
     98  dict_dates={}
     99  for f in files:
     100    if (f[0:2] == './'): f = f[2:] # PS - remove this part if present
     101    fn = os.path.basename(f)
     102    if fn[-1] != '*':
     103      timestamp = fn[tind:]
     104      year = int(timestamp[0:2])
     105      if year < 58:
     106        year += 2000
     107      else:
     108        year += 1900
     109      dtstring = str(year)+' '+timestamp[2:9]
     110      dict_dates[dtstring] = f
     111  dates = sorted(dict_dates.items())
     112  if len(dates) == 0:
     113    print 'no files found with this search string'
     114    print 'aborting. '
     115    sys.exit(0)
     116  else:
     117    print 'found ',len(dates),'files'
     118    #Prepare output files
     119    fout = file(os.path.join(avpath,'AVAILABLE'),'w')
     120    fout.write(AVAIL_head)
     121
     122  for i,date in enumerate(dates): # go through dates in ascending order
     123    f = date[1] # now we have the filename+path
     124    fn = os.path.basename(f)
     125    if fn[-1]!='*':
     126      timestamp = fn[tind:]
     127      year = int(timestamp[0:2])
     128      if year < 58:
     129        year += 2000
     130      else:
     131        year += 1900
     132      month = int(timestamp[2:4])
     133      day   = int(timestamp[4:6])
     134      hour  = int(timestamp[6:8])
     135      fileT = year*100 + int(month)
     136# PS: now check for irregular intervals     
     137      date = datetime.datetime(year,month,day,hour)
     138      if i == 2:
     139        if options.timeint == '':
     140          timeint = date - date1
     141        else:
     142          timeint = datetime.timedelta(0,3600*int(options.timeint)) 
     143        if timeint != date - date1:
     144          print 'WARNING - irregular interval',date - date1
     145          print date1,f1,'\n',date, f,'\n'
     146      elif i > 2:
     147        if timeint != date - date1:
     148          print 'WARNING - irregular interval',date - date1
     149          print date1,f1,'\n',date, f,'\n'
     150        if options.timeint == '': timeint = date - date1
     151      date1 = date 
     152      f1 = f
     153     
     154      if i%5000 == 0: print 'progress:', i, 'of', len(dates), f
     155     
     156      if fileT >= sy*100 + sm and fileT <= ey*100 + em :
     157
     158        relpath = os.path.relpath( os.path.dirname(f), avpath )
     159        f = os.path.join(relpath,fn)
     160
     161        if (f[0:2] == './'): f = f[2:] #  remove this part if present
     162
     163        if len(f) > 18: #PS
     164          if not warned:
     165            print 'WARNING: Flexpart can only read 18 chars in WF-name'
     166            print f, ' has ', len(f), ' characters!\n'
     167            warned = True
     168
     169        #This is the fortran format: (i8,1x,i6,2(6x,a18))
     170        string = "%s%s%s %s0000    %s    ON DISC\n" %\
     171         (year,str(month).zfill(2),str(day).zfill(2),str(hour).zfill(2),f.ljust(18))
     172        fout.write(string)
     173
     174  print 'Done: ',i+1 # as i starts with 0
     175  print 'Written:', os.path.join(avpath,'AVAILABLE')
     176  fout.close()
     177
     178if __name__ == '__main__':
     179  MODEL = {}
     180  MODEL['ECMWF'] = {'prefix':'EN', 'timeindex':2}
     181  MODEL['ERAI']  = {'prefix':'EI', 'timeindex':2}
     182  MODEL['EI']  = {'prefix':'EI', 'timeindex':2}
     183  MODEL['E5']  = {'prefix':'E5', 'timeindex':2}
     184  MODEL['EA']  = {'prefix':'EA', 'timeindex':2}
     185  MODEL['GFS']   = {'prefix':'GF', 'timeindex':2}
     186 
     187  models = '/'.join(MODEL.keys())
     188 
     189  try:
     190    start_time = time.time()
     191    today = datetime.datetime.now()
     192
     193    parser = optparse.OptionParser(
     194        formatter = optparse.TitledHelpFormatter(),
     195        usage = globals()['__doc__'])
     196
     197    parser.add_option ('-v', '--verbose', action = 'store_true',
     198        default = False, help = 'verbose output')
     199
     200    parser.add_option ('-s', '--startdate',
     201        dest = 'startdate',
     202        default = '197001',
     203        help = 'startdate YYYYMM integer')
     204
     205    parser.add_option ('-e', '--enddate',
     206        # setting default as TODAY
     207        dest = 'enddate',
     208        default = str(today.year) + str(today.month).zfill(2),
     209        #default = '200712',
     210        help = 'enddate YYYYMM integer')
     211
     212    parser.add_option ('-m', '--model',
     213        dest = 'model',
     214        default = 'ECMWF', help = models)
     215
     216    parser.add_option ('-p', '--path',
     217        dest = 'path',
     218        default = '.', help = 'path to be searched for windfields. Escape or quote * and ? ')
     219
     220    parser.add_option ('-a', '--avpath',
     221        dest = 'avpath',
     222        default = '.', help = 'path for AVAILABLE file ')
     223
     224    parser.add_option ('-i', '--interval',
     225        dest = 'timeint',
     226        default = '', help = 'expected time interval in h. If omitted, show every change')
     227
     228    (options, args) = parser.parse_args()
     229
     230    #QUERY['modelType'] = options.model
     231    #QUERY['start_date'] = options.startdate
     232    #QUERY['end_date'] = options.enddate
     233 
     234    #if len(args) < 1:
     235    #  parser.error ('missing argument')
     236
     237    if options.verbose: print time.asctime()
     238    exit_code = main()
     239    if exit_code is None:
     240      exit_code = 0
     241    if options.verbose: print time.asctime()
     242    if options.verbose: print 'TOTAL TIME IN MINUTES:',
     243    if options.verbose: print (time.time() - start_time) / 60.0
     244    sys.exit(exit_code)
     245  except KeyboardInterrupt, e: # Ctrl-C
     246    raise e
     247  except SystemExit, e: # sys.exit()
     248    raise e
     249  except Exception, e:
     250    print 'ERROR, UNEXPECTED EXCEPTION'
     251    print str(e)
     252    traceback.print_exc()
     253    os._exit(1)
    63254   
    64     WIND_dir = '/xnilu_wrk/flex_wrk/WIND_FIELDS'
    65    
    66     ## THE AVAILABE MODEL PRODUCTS (ultimately this comes from a DB)
    67     MODEL = {}
    68     MODEL['ECMWF'] = {'prefix':'EN', 'timeindex':2}
    69     MODEL['ERAI'] = {'prefix':'EI', 'timeindex':2}
    70     MODEL['GFS']   = {'prefix':'GF', 'timeindex':2}
    71 
    72     AVAIL_head = """DATE     TIME         FILENAME             SPECIFICATIONS\
    73     \nYYYYMMDD HHMISS\
    74     \n________ ______      __________________      __________________\n"""
    75     start_date = options.startdate
    76     sy = int(start_date[0:4])
    77     sm = int(start_date[4:6])
    78     end_date = options.enddate
    79     ey = int(end_date[0:4])
    80     em = int(end_date[4:6])
    81 #    print end_date
    82    
    83     #Get the directory information
    84     M = MODEL[options.model]
    85     prfx = M['prefix']
    86     t = options.path
    87     avpath = options.avpath
    88    
    89    
    90     #Loop through the files in the directories
    91     warned = False
    92     d = t #directory
    93 #    print d
    94     tind = M['timeindex'] #timestamp index
    95     searchstr = os.path.join(t,prfx)
    96     print 'searchstring:',searchstr
    97     files = glob.glob(searchstr+'*')
    98 #    print files
    99     dict_dates={}
    100     for f in files:
    101         if (f[0:2] == './'): f = f[2:] # PS - remove this part if present
    102         fn = os.path.basename(f)
    103         if fn[-1]!='*':
    104             timestamp = fn[tind:]
    105             year = int(timestamp[0:2])
    106             if year < 58:
    107                 year += 2000
    108             else:
    109                 year += 1900
    110             dtstring = str(year)+' '+timestamp[2:9]
    111             dict_dates[dtstring] = f
    112     dates=sorted(dict_dates.items())
    113     if len(dates) == 0:
    114       print 'no files found with this search string'
    115       print 'aborting. '
    116       sys.exit(0)
    117     else:
    118       print 'found ',len(dates),'files'
    119       #Prepare output files
    120       fout = file(os.path.join(avpath,'AVAILABLE'),'w')
    121       fout.write(AVAIL_head)
    122 
    123     for i,date in enumerate(dates): # go through dates in ascending order
    124         f = date[1] # now we have the filename+path
    125         fn = os.path.basename(f)
    126         if fn[-1]!='*':
    127             timestamp = fn[tind:]
    128             year = int(timestamp[0:2])
    129             if year < 58:
    130                 year += 2000
    131             else:
    132                 year += 1900
    133             month = int(timestamp[2:4])
    134             day = int(timestamp[4:6])
    135             hour = int(timestamp[6:8])
    136             fileT = year*100 + int(timestamp[2:4])
    137 # PS: check for irregular intervals           
    138             date = datetime.datetime(year,month,day,hour)
    139             if i == 2:
    140               if options.timeint == '':
    141                 timeint = date - date1
    142               else:
    143                 timeint = datetime.timedelta(0,3600*int(options.timeint)) 
    144                 if timeint != date - date1:
    145                   print 'WARNING - irregular interval',date - date1
    146                   print date1,f1,'\n',date, f,'\n'
    147             elif i > 2:
    148               if timeint != date - date1:
    149                 print 'WARNING - irregular interval',date - date1
    150                 print date1,f1,'\n',date, f,'\n'
    151                 if options.timeint == '': timeint = date - date1
    152             date1 = date 
    153             f1 = f
    154            
    155             if i%5000 == 0: print 'progress:', i, f
    156            
    157             if fileT <= ey*100 + em and fileT >= sy*100 + sm:
    158                 #if month <= em and month >= sm:
    159                     #print 'Match: %s' % fn
    160                 #This is the fortran format: (i8,1x,i6,2(6x,a18))
    161                 relpath = os.path.relpath(os.path.dirname(f),avpath)
    162                 f = os.path.join(relpath,fn)
    163                 if (f[0:2] == './'): f = f[2:] #  remove this part if present
    164 #                print fn,f
    165                 if len(f) > 18: #PS
    166                  if not warned:
    167                   print 'WARNING: Flexpart can only read 18 chars in WF-name'
    168                   print f, ' has ', len(f), ' characters!\n'
    169                   warned = True
    170                 string = "%s%s%s %s0000      %s      ON DISC\n" %\
    171                  (year,str(month).zfill(2),str(day).zfill(2),str(hour).zfill(2),f.ljust(18))
    172                 fout.write(string)
    173 
    174     print 'Done: ',i+1 # as i starts with 0
    175     print 'Written:', os.path.join(avpath,'AVAILABLE')
    176     fout.close()
    177 
    178 if __name__ == '__main__':
    179    
    180     try:
    181         start_time = time.time()
    182         today = datetime.datetime.now()
    183         parser = optparse.OptionParser(
    184                 formatter=optparse.TitledHelpFormatter(),
    185                 usage=globals()['__doc__'],
    186                 version='$Id: 0.001 $')
    187         parser.add_option ('-v', '--verbose', action='store_true',
    188                 default=False, help='verbose output')
    189         parser.add_option ('-s', '--startdate',
    190                 dest='startdate',
    191                 default='197001',
    192                 help='YYYYMM integer')
    193         parser.add_option ('-e', '--enddate',
    194                 # setting default as TODAY
    195                 dest='enddate',
    196                 default = str(today.year) + str(today.month).zfill(2),
    197                 #default='200712',
    198                 help='YYYYMM integer')
    199         parser.add_option ('-m', '--model',
    200                 dest='model',
    201                 default='ECMWF', help='ECMWF or ERAI or GFS')
    202         parser.add_option ('-p', '--path',
    203                 dest='path',
    204                 default='.', help='path to be searched. Escape or quote * and ? ')
    205         parser.add_option ('-a', '--avpath',
    206                 dest='avpath',
    207                 default='.', help='path for AVAILABLE file ')
    208         parser.add_option ('-i', '--interval',
    209                 dest='timeint',
    210                 default='', help='expected time interval in h. If omitted, show every change')
    211         (options, args) = parser.parse_args()
    212         #QUERY['modelType'] = options.model
    213         #QUERY['start_date'] = options.startdate
    214         #QUERY['end_date'] = options.enddate
    215  
    216         #if len(args) < 1:
    217         #    parser.error ('missing argument')
    218         if options.verbose: print time.asctime()
    219         exit_code = main()
    220         if exit_code is None:
    221             exit_code = 0
    222         if options.verbose: print time.asctime()
    223         if options.verbose: print 'TOTAL TIME IN MINUTES:',
    224         if options.verbose: print (time.time() - start_time) / 60.0
    225         sys.exit(exit_code)
    226     except KeyboardInterrupt, e: # Ctrl-C
    227         raise e
    228     except SystemExit, e: # sys.exit()
    229         raise e
    230     except Exception, e:
    231         print 'ERROR, UNEXPECTED EXCEPTION'
    232         print str(e)
    233         traceback.print_exc()
    234         os._exit(1)
    235        
    236    
     255 
    237256
    238257# vim:set sr et ts=4 sw=4 ft=python fenc=utf-8: // See Vim, :help 'modeline'