Index: python/EcFlexpart.py
===================================================================
--- python/EcFlexpart.py	(revision dda0e9deecb939467ef6d43f7ceb822d7eaee6b0)
+++ python/EcFlexpart.py	(revision 2fb99de9b96e65c8b5caa85c276ad8e0be540ed0)
@@ -102,4 +102,5 @@
 
 # software specific classes and modules from flex_extract
+import _config
 from GribTools import GribTools
 from tools import init128, to_param_id, silent_remove, product, my_error
@@ -204,10 +205,5 @@
         self.accuracy = c.accuracy
         self.level = c.level
-
-        if c.levelist:
-            self.levelist = c.levelist
-        else:
-            self.levelist = '1/to/' + c.level
-
+        self.expver = c.expver
         # for gaussian grid retrieval
         self.glevelist = '1/to/' + c.level
@@ -217,14 +213,4 @@
         else:
             self.gaussian = ''
-
-        if hasattr(c, 'expver') and c.expver:
-            self.expver = c.expver
-        else:
-            self.expver = '1'
-
-        if hasattr(c, 'number') and c.number:
-            self.number = c.number
-        else:
-            self.number = '0'
 
         if 'N' in c.grid:  # Gaussian output grid
@@ -250,5 +236,4 @@
         # 4) Download also data for WRF
 
-
         # Different grids need different retrievals
         # SH = Spherical Harmonics, GG = Gaussian Grid,
@@ -274,11 +259,14 @@
             self.params['OG__ML'] = ['T/Q', 'ML', self.levelist, self.grid]
 
-            if c.gauss == '0' and c.eta == '1':
+            #if c.gauss == '0' and c.eta == '1':
+            if not c.gauss and c.eta:
                 # the simplest case
                 self.params['OG__ML'][0] += '/U/V/77'
-            elif c.gauss == '0' and c.eta == '0':
+            #elif c.gauss == '0' and c.eta == '0':
+            elif not c.gauss and not c.eta:
             # this is not recommended (inaccurate)
                 self.params['OG__ML'][0] += '/U/V'
-            elif c.gauss == '1' and c.eta == '0':
+            #elif c.gauss == '1' and c.eta == '0':
+            elif c.gauss and not c.eta:
                 # this is needed for data before 2008, or for reanalysis data
                 self.params['GG__SL'] = ['Q', 'ML', '1', \
@@ -286,6 +274,6 @@
                 self.params['SH__ML'] = ['U/V/D', 'ML', self.glevelist, 'OFF']
             else:
-                print 'Warning: This is a very costly parameter combination, \
-                       use only for debugging!'
+                print('Warning: This is a very costly parameter combination, \
+                      use only for debugging!')
                 self.params['GG__SL'] = ['Q', 'ML', '1', \
                                          '{}'.format((int(self.resol) + 1) / 2)]
@@ -293,13 +281,13 @@
                                          '{}'.format((int(self.resol) + 1) / 2)]
 
-            if hasattr(c, 'omega') and c.omega == '1':
+            if c.omega:
                 self.params['OG__ML'][0] += '/W'
 
             # add cloud water content if necessary
-            if hasattr(c, 'cwc') and c.cwc == '1':
+            if c.cwc:
                 self.params['OG__ML'][0] += '/CLWC/CIWC'
 
             # add vorticity and geopotential height for WRF if necessary
-            if hasattr(c, 'wrf') and c.wrf == '1':
+            if c.wrf:
                 self.params['OG__ML'][0] += '/Z/VO'
                 if '/D' not in self.params['OG__ML'][0]:
@@ -367,7 +355,8 @@
             f.write('&NAMGEN\n')
             f.write(',\n  '.join(['maxl = ' + str(maxl), 'maxb = ' + str(maxb),
-                                  'mlevel = ' + self.level,
-                                  'mlevelist = ' + '"' + self.levelist + '"',
-                                  'mnauf = ' + self.resol,
+                                  'mlevel = ' + str(self.level),
+                                  'mlevelist = ' + '"' + str(self.levelist)
+                                                 + '"',
+                                  'mnauf = ' + str(self.resol),
                                   'metapar = ' + '77',
                                   'rlo0 = ' + str(area[1]),
@@ -375,11 +364,11 @@
                                   'rla0 = ' + str(area[2]),
                                   'rla1 = ' + str(area[0]),
-                                  'momega = ' + c.omega,
-                                  'momegadiff = ' + c.omegadiff,
-                                  'mgauss = ' + c.gauss,
-                                  'msmooth = ' + c.smooth,
-                                  'meta = ' + c.eta,
-                                  'metadiff = ' + c.etadiff,
-                                  'mdpdeta = ' + c.dpdeta]))
+                                  'momega = ' + str(c.omega),
+                                  'momegadiff = ' + str(c.omegadiff),
+                                  'mgauss = ' + str(c.gauss),
+                                  'msmooth = ' + str(c.smooth),
+                                  'meta = ' + str(c.eta),
+                                  'metadiff = ' + str(c.etadiff),
+                                  'mdpdeta = ' + str(c.dpdeta)]))
 
             f.write('\n/\n')
@@ -387,5 +376,5 @@
         return
 
-    def retrieve(self, server, dates, inputdir='.'):
+    def retrieve(self, server, dates, request, inputdir='.'):
         '''
         @Description:
@@ -437,5 +426,5 @@
                     pass
                 if pk == 'OG_OROLSM__SL':
-                    if oro is False:
+                    if not oro:
                         mfstream = 'OPER'
                         mftype = 'AN'
@@ -457,5 +446,5 @@
 
     # ------  on demand path  --------------------------------------------------
-                if self.basetime is None:
+                if not self.basetime:
                     MR = MarsRetrieval(self.server,
                                        marsclass=self.marsclass,
@@ -477,6 +466,15 @@
                                        param=pv[0])
 
-                    MR.display_info()
-                    MR.data_retrieve()
+                    if request == 0:
+                        MR.display_info()
+                        MR.data_retrieve()
+                    elif request == 1:
+                        MR.print_info()
+                    elif request == 2:
+                        MR.print_info()
+                        MR.display_info()
+                        MR.data_retrieve()
+                    else:
+                        print('Failure')
     # ------  operational path  ------------------------------------------------
                 else:
@@ -566,4 +564,5 @@
 
                             MR.display_info()
+
                             MR.data_retrieve()
                         # --------------  non flux data ------------------------
@@ -662,5 +661,8 @@
                             MR.data_retrieve()
 
-        print "MARS retrieve done... "
+        if request == 0 or request == 2:
+            print('MARS retrieve done ... ')
+        elif request == 1:
+            print('MARS request printed ...')
 
         return
@@ -702,18 +704,18 @@
         '''
 
-        print '\n\nPostprocessing:\n Format: {}\n'.format(c.format)
-
-        if c.ecapi is False:
+        print('\n\nPostprocessing:\n Format: {}\n'.format(c.format))
+
+        if not c.ecapi:
             print('ecstorage: {}\n ecfsdir: {}\n'.
                   format(c.ecstorage, c.ecfsdir))
-            if not hasattr(c, 'gateway'):
-                c.gateway = os.getenv('GATEWAY')
-            if not hasattr(c, 'destination'):
-                c.destination = os.getenv('DESTINATION')
+            #if not hasattr(c, 'gateway'):
+            #    c.gateway = os.getenv('GATEWAY')
+            #if not hasattr(c, 'destination'):
+            #    c.destination = os.getenv('DESTINATION')
             print('ectrans: {}\n gateway: {}\n destination: {}\n '
                   .format(c.ectrans, c.gateway, c.destination))
 
-        print 'Output filelist: \n'
-        print self.outputfilelist
+        print('Output filelist: \n')
+        print(self.outputfilelist)
 
         if c.format.lower() == 'grib2':
@@ -724,5 +726,5 @@
                 p = subprocess.check_call(['mv', ofile + '_2', ofile])
 
-        if int(c.ectrans) == 1 and c.ecapi is False:
+        if c.ectrans and not c.ecapi:
             for ofile in self.outputfilelist:
                 p = subprocess.check_call(['ectrans', '-overwrite', '-gateway',
@@ -731,5 +733,5 @@
                 #print('ectrans:', p)
 
-        if int(c.ecstorage) == 1 and c.ecapi is False:
+        if c.ecstorage and not c.ecapi:
             for ofile in self.outputfilelist:
                 p = subprocess.check_call(['ecp', '-o', ofile,
@@ -742,5 +744,5 @@
         # prepare environment for the grib2flexpart run
         # to convert grib to flexpart binary
-        if c.grib2flexpart == '1':
+        if c.grib2flexpart:
 
             # generate AVAILABLE file
@@ -858,5 +860,5 @@
 
         index_keys = ["date", "time", "step"]
-        indexfile = c.inputdir + "/date_time_stepRange.idx"
+        indexfile = os.path.join(c.inputdir, _config.FILE_GRIB_INDEX)
         silent_remove(indexfile)
         grib = GribTools(inputfiles.files)
@@ -868,5 +870,5 @@
         for key in index_keys:
             index_vals.append(grib_index_get(iid, key))
-            print index_vals[-1]
+            print(index_vals[-1])
             # index_vals looks for example like:
             # index_vals[0]: ('20171106', '20171107', '20171108') ; date
@@ -878,7 +880,11 @@
 
         for prod in product(*index_vals):
+            # e.g. prod = ('20170505', '0', '12')
+            #             (  date    ,time, step)
+            # per date e.g. time = 0, 1200
+            # per time e.g. step = 3, 6, 9, 12
             # flag for Fortran program CONVERT2 and file merging
             convertFlag = False
-            print 'current prod: ', prod
+            print('current prod: ', prod)
             # e.g. prod = ('20170505', '0', '12')
             #             (  date    ,time, step)
@@ -902,6 +908,7 @@
                 # they are just valid for a single product
                 for k, f in fdict.iteritems():
-                    silent_remove(c.inputdir + "/fort." + k)
-                    fdict[k] = open(c.inputdir + '/fort.' + k, 'w')
+                    fortfile = os.path.join(c.inputdir, 'fort.' + k)
+                    silent_remove(fortfile)
+                    fdict[k] = open(fortfile, 'w')
 
                 cdate = str(grib_get(gid, 'date'))
@@ -931,21 +938,18 @@
                     if timestamp < slimit or timestamp > elimit:
                         continue
+            else:
+                pass
 
             try:
-                if c.wrf == '1':
-                    if 'olddate' not in locals():
-                        fwrf = open(c.outputdir + '/WRF' + cdate +
-                                    '.{:0>2}'.format(time) + '.000.grb2', 'w')
+                if c.wrf:
+                    if 'olddate' not in locals() or cdate != olddate:
+                        fwrf = open(os.path.join(c.outputdir,
+                                    'WRF' + cdate + '.{:0>2}'.format(time) +
+                                    '.000.grb2'), 'w')
                         olddate = cdate[:]
-                    else:
-                        if cdate != olddate:
-                            fwrf = open(c.outputdir + '/WRF' + cdate +
-                                        '.{:0>2}'.format(time) + '.000.grb2',
-                                        'w')
-                            olddate = cdate[:]
             except AttributeError:
                 pass
 
-            # helper variable to remember which fields are already used.
+            # helper variable to remember which fields were already used.
             savedfields = []
             while 1:
@@ -958,5 +962,5 @@
                 # Specific humidity (Q.grb) is used as a template only
                 # so we need the first we "meet"
-                    with open(c.inputdir + '/fort.18', 'w') as fout:
+                    with open(os.path.join(c.inputdir, 'fort.18'), 'w') as fout:
                         grib_write(gid, fout)
                 elif paramId == 131 or paramId == 132:
@@ -971,5 +975,5 @@
                     grib_write(gid, fdict['13'])
                 elif  paramId in [129, 138, 155] and levtype == 'hybrid' \
-                        and c.wrf == '1':
+                        and c.wrf:
                     pass
                 elif paramId == 246 or paramId == 247:
@@ -991,14 +995,15 @@
                         savedfields.append(paramId)
                     else:
-                        print 'duplicate ' + str(paramId) + ' not written'
+                        print('duplicate ' + str(paramId) + ' not written')
 
                 try:
-                    if c.wrf == '1':
-                        if levtype == 'hybrid': # model layer
-                            if paramId in [129, 130, 131, 132, 133, 138, 155]:
-                                grib_write(gid, fwrf)
-                        else: # sfc layer
-                            if paramId in wrfpars:
-                                grib_write(gid, fwrf)
+                    if c.wrf:
+                        # model layer
+                        if levtype == 'hybrid' and \
+                           paramId in [129, 130, 131, 132, 133, 138, 155]:
+                            grib_write(gid, fwrf)
+                        # sfc layer
+                        elif paramId in wrfpars:
+                            grib_write(gid, fwrf)
                 except AttributeError:
                     pass
@@ -1014,8 +1019,8 @@
                 pwd = os.getcwd()
                 os.chdir(c.inputdir)
-                if os.stat('fort.21').st_size == 0 and int(c.eta) == 1:
-                    print 'Parameter 77 (etadot) is missing, most likely it is \
-                           not available for this type or date/time\n'
-                    print 'Check parameters CLASS, TYPE, STREAM, START_DATE\n'
+                if os.stat('fort.21').st_size == 0 and c.eta:
+                    print('Parameter 77 (etadot) is missing, most likely it is \
+                           not available for this type or date/time\n')
+                    print('Check parameters CLASS, TYPE, STREAM, START_DATE\n')
                     my_error(c.mailfail, 'fort.21 is empty while parameter eta \
                              is set to 1 in CONTROL file')
@@ -1023,11 +1028,10 @@
                 # create the corresponding output file fort.15
                 # (generated by CONVERT2) + fort.16 (paramId 167 and 168)
-                p = subprocess.check_call(
-                    [os.path.expandvars(os.path.expanduser(c.exedir)) +
-                     '/CONVERT2'], shell=True)
+                p = subprocess.check_call([os.path.join(c.exedir, 'CONVERT2')],
+                                          shell=True)
                 os.chdir(pwd)
 
                 # create final output filename, e.g. EN13040500 (ENYYMMDDHH)
-                fnout = c.inputdir + '/' + c.prefix
+                fnout = os.path.join(c.inputdir, c.prefix)
                 if c.maxstep > 12:
                     suffix = cdate[2:8] + '.{:0>2}'.format(time/100) + \
@@ -1036,5 +1040,5 @@
                     suffix = cdateH[2:10]
                 fnout += suffix
-                print "outputfile = " + fnout
+                print("outputfile = " + fnout)
                 self.outputfilelist.append(fnout) # needed for final processing
 
@@ -1044,5 +1048,5 @@
                     c.inputdir + '/OG_OROLSM__SL.*.' + c.ppid + '*')[0])
                 fluxfile = 'flux' + cdate[0:2] + suffix
-                if c.cwc != '1':
+                if not c.cwc:
                     flist = ['fort.15', fluxfile, 'fort.16', orolsm]
                 else:
@@ -1051,13 +1055,16 @@
                 with open(fnout, 'wb') as fout:
                     for f in flist:
+                        shutil.copyfileobj(open(os.path.join(c.inputdir, f),
+                                                'rb'), fout)
+
+                if c.omega:
+                    with open(os.path.join(c.outputdir, 'OMEGA'), 'wb') as fout:
                         shutil.copyfileobj(
-                            open(c.inputdir + '/' + f, 'rb'), fout)
-
-                if c.omega == '1':
-                    with open(c.outputdir + '/OMEGA', 'wb') as fout:
-                        shutil.copyfileobj(
-                            open(c.inputdir + '/fort.25', 'rb'), fout)
-
-        if hasattr(c, 'wrf') and c.wrf == '1':
+                            open(os.path.join(c.inputdir, 'fort.25'),
+                                 'rb'), fout)
+            else:
+                pass
+
+        if c.wrf:
             fwrf.close()
 
@@ -1102,5 +1109,5 @@
         pars = to_param_id(self.params['OG_acc_SL'][0], table128)
         index_keys = ["date", "time", "step"]
-        indexfile = c.inputdir + "/date_time_stepRange.idx"
+        indexfile = os.path.join(c.inputdir, _config.FILE_GRIB_INDEX)
         silent_remove(indexfile)
         grib = GribTools(inputfiles.files)
@@ -1112,5 +1119,5 @@
         for key in index_keys:
             key_vals = grib_index_get(iid, key)
-            print key_vals
+            print(key_vals)
             # have to sort the steps for disaggregation,
             # therefore convert to int first
@@ -1133,5 +1140,5 @@
             stepsdict[str(p)] = []
 
-        print 'maxstep: ', c.maxstep
+        print('maxstep: ', c.maxstep)
 
         for prod in product(*index_vals):
@@ -1140,8 +1147,13 @@
             # per date e.g. time = 0, 1200
             # per time e.g. step = 3, 6, 9, 12
+            print('current prod: ', prod)
             for i in range(len(index_keys)):
                 grib_index_select(iid, index_keys[i], prod[i])
 
+            # get first id from current product
             gid = grib_new_from_index(iid)
+
+            # if there is data for this product combination
+            # prepare some date and time parameter before reading the data
             if gid is not None:
                 cdate = grib_get(gid, 'date')
@@ -1161,26 +1173,29 @@
 
             if c.maxstep > 12:
-                fnout = c.inputdir + '/flux' + \
-                    sdate.strftime('%Y%m%d') + '.{:0>2}'.format(time/100) + \
-                    '.{:0>3}'.format(step-2*int(c.dtime))
-                gnout = c.inputdir + '/flux' + \
-                    sdate.strftime('%Y%m%d') + '.{:0>2}'.format(time/100) + \
-                    '.{:0>3}'.format(step-int(c.dtime))
-                hnout = c.inputdir + '/flux' + \
-                    sdate.strftime('%Y%m%d') + '.{:0>2}'.format(time/100) + \
-                    '.{:0>3}'.format(step)
-                g = open(gnout, 'w')
-                h = open(hnout, 'w')
+                fnout = os.path.join(c.inputdir, 'flux' +
+                                     sdate.strftime('%Y%m%d') +
+                                     '.{:0>2}'.format(time/100) +
+                                     '.{:0>3}'.format(step-2*int(c.dtime)))
+                gnout = os.path.join(c.inputdir, 'flux' +
+                                     sdate.strftime('%Y%m%d') +
+                                     '.{:0>2}'.format(time/100) +
+                                     '.{:0>3}'.format(step-int(c.dtime)))
+                hnout = os.path.join(c.inputdir, 'flux' +
+                                     sdate.strftime('%Y%m%d') +
+                                     '.{:0>2}'.format(time/100) +
+                                     '.{:0>3}'.format(step))
             else:
-                fnout = c.inputdir + '/flux' + fdate.strftime('%Y%m%d%H')
-                gnout = c.inputdir + '/flux' + (fdate +
-                                                timedelta(hours=int(c.dtime))
-                                               ).strftime('%Y%m%d%H')
-                hnout = c.inputdir + '/flux' + sdates.strftime('%Y%m%d%H')
-                g = open(gnout, 'w')
-                h = open(hnout, 'w')
-
-            print "outputfile = " + fnout
-            f = open(fnout, 'w')
+                fnout = os.path.join(c.inputdir, 'flux' +
+                                     fdate.strftime('%Y%m%d%H'))
+                gnout = os.path.join(c.inputdir, 'flux' +
+                                     (fdate + timedelta(hours=int(c.dtime))).
+                                     strftime('%Y%m%d%H'))
+                hnout = os.path.join(c.inputdir, 'flux' +
+                                     sdates.strftime('%Y%m%d%H'))
+
+            print("outputfile = " + fnout)
+            f_handle = open(fnout, 'w')
+            g_handle = open(gnout, 'w')
+            h_handle = open(hnout, 'w')
 
             # read message for message and store relevant data fields
@@ -1243,5 +1258,5 @@
                             grib_set(gid, 'date', fdate.year*10000 +
                                      fdate.month*100+fdate.day)
-                        grib_write(gid, f)
+                        grib_write(gid, f_handle)
 
                         if c.basetime is not None:
@@ -1270,5 +1285,5 @@
                                      truedatetime.month * 100 +
                                      truedatetime.day)
-                            grib_write(gid, h)
+                            grib_write(gid, h_handle)
 
                             #values = (svdp[1]+svdp[2])/2.
@@ -1285,5 +1300,5 @@
                                      truedatetime.day)
                             grib_set_values(gid, values)
-                            grib_write(gid, g)
+                            grib_write(gid, g_handle)
 
                     grib_release(gid)
@@ -1291,7 +1306,7 @@
                     gid = grib_new_from_index(iid)
 
-            f.close()
-            g.close()
-            h.close()
+            f_handle.close()
+            g_handle.close()
+            h_handle.close()
 
         grib_index_release(iid)
