Index: Source/Python/install.py
===================================================================
--- Source/Python/install.py	(revision bbbe1e1e7fd5d92f8cbb0005f1fc0255c6cc75a1)
+++ Source/Python/install.py	(revision 80281769fb906cbf4c8c64994964373c213ce779)
@@ -376,5 +376,5 @@
                                       'CONTROL*').files]
     testfiles = [os.path.relpath(x, ecd)
-                 for x in UioFiles(_config.PATH_REL_TEST, '*').files]
+                 for x in UioFiles(_config.PATH_REL_TEST+"/Installation", '*').files]
     tempfiles = [os.path.relpath(x, ecd)
                  for x in UioFiles(_config.PATH_REL_TEMPLATES, '*.template').files]
@@ -387,5 +387,6 @@
     makefiles = [os.path.relpath(x, ecd)
                  for x in UioFiles(_config.PATH_REL_FORTRAN_SRC, 'makefile*').files]
-    jobdir = [_config.PATH_REL_JOBSCRIPTS]
+    jobdir = [os.path.relpath(x, ecd)
+               for x in UioFiles(_config.PATH_REL_JOBSCRIPTS, '*.md').files]
 
     # concatenate single lists to one for a better looping
Index: Source/Pythontest/TestEcFlexpart.py
===================================================================
--- Source/Pythontest/TestEcFlexpart.py	(revision ba9923062b4af98cbdd20c4101c305766fc728f5)
+++ 	(revision )
@@ -1,56 +1,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-
-import sys
-import os
-import inspect
-import pytest
-
-sys.path.append('../python')
-import _config
-from classes.EcFlexpart import EcFlexpart
-from classes.ControlFile import ControlFile
-from mods.tools import silent_remove
-
-class TestEcFlexpart():
-    '''
-    '''
-
-    def test_write_namelist(self):
-        import filecmp
-
-        control_file = os.path.join(_config.PATH_TEST_DIR,
-                                        'TestData',
-                                        'CONTROL.temp')
-        c = ControlFile(control_file)
-        flexpart = EcFlexpart(c)
-
-        c.inputdir = 'TestData'
-
-        # comparison file
-        testfile = os.path.join(_config.PATH_TEST_DIR,
-                                'TestData',
-                                'convert.nl.test')
-
-        # create
-        flexpart.write_namelist(c)
-
-        finalfile = os.path.join(c.inputdir, _config.FILE_NAMELIST)
-        assert filecmp.cmp(testfile, finalfile, shallow=False)
-
-        # delete test file
-        silent_remove(finalfile)
-
-
-
-
-# these functions should test the output and compare the results with an output
-# of the old version and check if there are no differences!!!
-# also check for errors?! Or check if it works for alle datasets and private public etc
-
-    #    - process_output
-    #    - create
-    #    - deacc_fluxes
-    #    - retrieve
-
Index: Source/Pythontest/TestInput.py
===================================================================
--- Source/Pythontest/TestInput.py	(revision 76c37a99c13dbe456f0aa5adcf7439778e84a4c3)
+++ 	(revision )
@@ -1,165 +1,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-import os
-import sys
-import pytest
-
-sys.path.append('../Python')
-from Classes.ControlFile import ControlFile
-from Mods.tools import get_cmdline_args
-
-
-class TestInput():
-    '''
-    Test class to test the reading of commandline arguments and
-    control file.
-    '''
-    # ToDo
-    # create more tests for input
-    # 1. nur controlfile reading
-    # 2. check of parameter
-
-    @classmethod
-    def setup_class(self):
-        # Default values for ArgumentParser
-        self.args = {'start_date':None,
-                     'end_date':None,
-                     'date_chunk':None,
-                     'basetime':None,
-                     'step':None,
-                     'levelist':None,
-                     'area':None,
-                     'inputdir':None,
-                     'outputdir':None,
-                     'flexpart_root_scripts':None,
-                     'ppid':None,
-                     'job_template':'job.temp',
-                     'queue':None,
-                     'controlfile':'CONTROL.test',
-                     'debug':0,
-                     }
-        #sys.argv = ['dummy.py', '--start_date=20180101', '--debug=1',
-        #            '--step=0/to/11/BY/3', '--area=20./20./0./90.']
-        sys.argv = ['dummy.py', '--start_date=20180101']
-
-        self.args = get_cmdline_args()
-
-        self.c = ControlFile('../../Testing/Regression/Unit/Testfiles/CONTROL.test')
-
-        self.c.assign_args_to_control(self.args)
-
-        self.c.check_conditions()
-
-
-    def test_args_reading(self):
-
-        sys.argv = ['dummy.py', '--start_date=20180101', '--debug=1',
-                    '--step=0/to/11/BY/3', '--area=20./20./0./90.']
-
-        arguments = get_cmdline_args()
-
-        args_exp = {'start_date':'20180101',
-                    'end_date':None,
-                    'date_chunk':None,
-                    'basetime':None,
-                    'step':'0/to/11/BY/3',
-                    'levelist':None,
-                    'area':'20./20./0./90.',
-                    'inputdir':None,
-                    'outputdir':None,
-                    'flexpart_root_scripts':None,
-                    'ppid':None,
-                    'job_template':'job.temp',
-                    'queue':None,
-                    'controlfile':'CONTROL.test',
-                    'debug':1,
-                    }
-
-        assert vars(arguments) == args_exp
-
-
-    def test_args_assignment(self):
-
-        import collections
-
-        # expected parametervalue:
-        exp_dict = {
-                    'accuracy': '16',
-                    'addpar': ['186', '187', '188', '235', '139', '39'],
-                    'area': None,
-                    'basetime': None,
-                    'controlfile': 'CONTROL.test',
-                    'cwc': 0,
-                    'date_chunk': 3,
-                    'debug': 0,
-                    'destination': None,
-                    'dpdeta': '1',
-                    'dtime': '3',
-                    'ecfsdir': 'ectmp:/${USER}/econdemand/',
-                    'ecgid': None,
-                    'ecmwfdatadir': '/raid60/nas/tmc/Anne/Interpolation/flexextract/flexextract/python/../',
-                    'ecstorage': '0',
-                    'ectrans': '1',
-                    'ecuid': None,
-                    'end_date': '20180101',
-                    'eta': '0',
-                    'etadiff': '0',
-                    'etapar': 77,
-                    'exedir': '/raid60/nas/tmc/Anne/Interpolation/flexextract/flexextract/python/../src/',
-                    'expver': '1',
-                    'flexpart_root_scripts': '/raid60/nas/tmc/Anne/Interpolation/flexextract/flexextract/python/../',
-                    'format': 'GRIB1',
-                    'gateway': None,
-                    'gauss': '1',
-                    'grib2flexpart': '0',
-                    'grid': '5000',
-                    'inputdir': '../work',
-                    'job_template': 'job.temp',
-                    'left': '-15000',
-                    'level': '60',
-                    'levelist': '55/to/60',
-                    'lower': '30000',
-                    'mailfail': ['${USER}'],
-                    'mailops': ['${USER}'],
-                    'makefile': None,
-                    'marsclass': 'EI',
-                    'maxstep': 11,
-                    'number': 'OFF',
-                    'omega': '0',
-                    'omegadiff': '0',
-                    'outputdir': '../work',
-                    'prefix': 'EI',
-                    'resol': '63',
-                    'right': '45000',
-                    'smooth': '0',
-                    'start_date': '20180101',
-                    'step': ['00', '01', '02', '03', '04', '05', '00', '07', '08', '09', '10', '11', '00', '01', '02', '03', '04', '05', '00', '07', '08', '09', '10', '11'],
-                    'stream': 'OPER',
-                    'target': None,
-                    'time': ['00', '00', '00', '00', '00', '00', '06', '00', '00', '00', '00', '00', '12', '12', '12', '12', '12', '12', '18', '12', '12', '12', '12', '12'],
-                    'type': ['AN', 'FC', 'FC', 'FC', 'FC', 'FC', 'AN', 'FC', 'FC', 'FC', 'FC', 'FC', 'AN', 'FC', 'FC', 'FC', 'FC', 'FC', 'AN', 'FC', 'FC', 'FC', 'FC', 'FC'],
-                    'upper': '75000',
-                    'wrf': 0}
-
-        exp_dict = collections.OrderedDict(sorted(exp_dict.items()))
-        cdict = collections.OrderedDict(sorted(vars(self.c).items()))
-
-        # remove content which isn't comparable for different users
-        # or different operating systems
-        del cdict['ecfsdir_expanded']
-        del cdict['mailops_expanded']
-        del cdict['mailfail_expanded']
-
-        #print 'cdict\n', cdict
-        #print 'exp_dict\n', exp_dict
-
-        #assert cdict == exp_dict
-        assert cdict == exp_dict
-
-        return
-
-    @classmethod
-    def teardown_class(self):
- 
-        return
Index: Source/Pythontest/TestInstall.py
===================================================================
--- Source/Pythontest/TestInstall.py	(revision 76c37a99c13dbe456f0aa5adcf7439778e84a4c3)
+++ Source/Pythontest/TestInstall.py	(revision 80281769fb906cbf4c8c64994964373c213ce779)
@@ -19,4 +19,5 @@
 import pytest
 from mock import patch
+import mock
 
 sys.path.append('../Python')
@@ -53,7 +54,8 @@
 
 
-    @patch('tarfile.open', side_effect=[subprocess.CalledProcessError(1,'test'),
-                                        OSError(errno.EEXIST)])
+    @patch('tarfile.open', side_effect=[tarfile.TarError, OSError])
     def test_fail_mk_tarball_local(self, mock_open):
+        import tarfile
+       # mock_open.side_effekt = tarfile.TarError
         ecd = _config.PATH_FLEXEXTRACT_DIR + os.path.sep
         # create test tarball and list its content files
@@ -62,4 +64,5 @@
         with pytest.raises(SystemExit):
             mk_tarball(ecd + tarballname, 'local')
+
 
     def test_success_mk_tarball_local(self):
@@ -119,12 +122,7 @@
         # list comparison files for tarball content
         tar_test_dir = os.path.join(self.testdir, 'InstallTar')
-        cmp_dir = _config.FLEXEXTRACT_DIRNAME + '_ecgate'
-        tar_test_fedir = os.path.join(tar_test_dir, cmp_dir)
-        comparison_list = []
-        for path, subdirs, files in os.walk(tar_test_fedir):
-            for name in files:
-                if 'tar' not in name:
-                    comparison_list.append(os.path.relpath(
-                        os.path.join(path, name), tar_test_fedir))
+        tarballname = _config.FLEXEXTRACT_DIRNAME + '_ecgate.tar'
+        with tarfile.open(os.path.join(tar_test_dir, tarballname), 'r') as tar_handle:
+            comparison_list = tar_handle.getnames()
 
         # untar in test directory
@@ -132,5 +130,4 @@
         make_dir(test_dir)
         os.chdir(test_dir)
-        tarballname = _config.FLEXEXTRACT_DIRNAME + '_ecgate.tar'
         un_tarball(os.path.join(tar_test_dir, tarballname))
         tarfiles_list = []
@@ -148,12 +145,7 @@
         # list comparison files for tarball content
         tar_test_dir = os.path.join(self.testdir, 'InstallTar')
-        cmp_dir = _config.FLEXEXTRACT_DIRNAME + '_local'
-        tar_test_fedir = os.path.join(tar_test_dir, cmp_dir)
-        comparison_list = []
-        for path, subdirs, files in os.walk(tar_test_fedir):
-            for name in files:
-                if 'tar' not in name:
-                    comparison_list.append(os.path.relpath(
-                        os.path.join(path, name), tar_test_fedir))
+        tarballname = _config.FLEXEXTRACT_DIRNAME + '_local.tar'
+        with tarfile.open(os.path.join(tar_test_dir, tarballname), 'r') as tar_handle:
+            comparison_list = tar_handle.getnames()
 
         # untar in test directory
@@ -161,5 +153,4 @@
         make_dir(test_dir)
         os.chdir(test_dir)
-        tarballname = _config.FLEXEXTRACT_DIRNAME + '_local.tar'
         un_tarball(os.path.join(tar_test_dir, tarballname))
         tarfiles_list = []
@@ -168,5 +159,5 @@
                 tarfiles_list.append(os.path.relpath(
                     os.path.join(path, name), test_dir))
-
+        		   
         # test for equality
         assert sorted(tarfiles_list) == sorted(comparison_list)
@@ -205,12 +196,4 @@
                             'user@destination')
 
-    @patch('__builtin__.open', side_effect=[OSError(errno.EPERM)])
-    def test_fail_open_mk_env_vars(self, mock_open):
-        with pytest.raises(SystemExit):
-            mk_env_vars('testuser',
-                        'testgroup',
-                        'gateway.test.ac.at',
-                        'user@destination')
-
     @patch('_config.FILE_INSTALL_COMPILEJOB', _config_test.PATH_TESTFILES_DIR+'/compilejob_test.ksh')
     def test_success_mk_compilejob(self):
@@ -221,5 +204,4 @@
 
         mk_compilejob('Makefile.TEST',
-                      '',
                       'testuser',
                       'testgroup',
@@ -236,5 +218,4 @@
         with pytest.raises(SystemExit):
             mk_compilejob('Makefile.TEST',
-                          '',
                           'testuser',
                           'testgroup',
@@ -246,29 +227,27 @@
             with pytest.raises(SystemExit):
                 mk_compilejob('Makefile.TEST',
-                              '',
                               'testuser',
                               'testgroup',
                               'fp_root_test_path')
 
-    @patch('__builtin__.open', side_effect=[OSError(errno.EPERM)])
-    def test_fail_open_mk_compilejob(self, mock_open):
-        with pytest.raises(SystemExit):
-            mk_compilejob('Makefile.TEST',
-                          '',
-                          'testuser',
-                          'testgroup',
-                          'fp_root_test_path')
-
-    @patch('_config.TEMPFILE_JOB', _config_test.PATH_TESTFILES_DIR+'/job_temp.test_test')
+#    @patch('builtins.open', side_effect=[OSError(errno.EPERM)])
+#    def test_fail_open_mk_compilejob(self, mock_open):
+#        with pytest.raises(SystemExit):
+#            mk_compilejob('Makefile.TEST',
+#                          'testuser',
+#                          'testgroup',
+#                          'fp_root_test_path')
+
+    @patch('_config.TEMPFILE_JOB', _config_test.PATH_TESTFILES_DIR+'/submitscript.template.test.comp')
     def test_success_mk_job_template(self):
         import filecmp
 
         testfile = os.path.join(self.testfilesdir,
-                                'job.temp.test')
+                                'submitscript.template.test')
 
         mk_job_template('testuser',
                         'testgroup',
-                        'gateway.test.ac.at',
-                        'dest@generic',
+#                        'gateway.test.ac.at',
+#                        'dest@generic',
                         'fp_root_test_path')
 
@@ -284,6 +263,6 @@
             mk_job_template('testuser',
                             'testgroup',
-                            'gateway.test.ac.at',
-                            'dest@generic',
+#                            'gateway.test.ac.at',
+#                            'dest@generic',
                             'fp_root_test_path')
 
@@ -294,16 +273,16 @@
                 mk_job_template('testuser',
                                 'testgroup',
-                                'gateway.test.ac.at',
-                                'dest@generic',
+ #                               'gateway.test.ac.at',
+ #                               'dest@generic',
                                 'fp_root_test_path')
 
-    @patch('__builtin__.open', side_effect=[OSError(errno.EPERM)])
-    def test_fail_open_mk_job_template(self, mock_open):
-        with pytest.raises(SystemExit):
-            mk_job_template('testuser',
-                            'testgroup',
-                            'gateway.test.ac.at',
-                            'dest@generic',
-                            'fp_root_test_path')
+#    @patch('builtins.open', side_effect=[OSError(errno.EPERM)])
+#    def test_fail_open_mk_job_template(self, mock_open):
+#        with pytest.raises(SystemExit):
+#            mk_job_template('testuser',
+#                            'testgroup',
+#                            'gateway.test.ac.at',
+#                            'dest@generic',
+#                            'fp_root_test_path')
 
     @classmethod
Index: Source/Pythontest/TestUIOFiles.py
===================================================================
--- Source/Pythontest/TestUIOFiles.py	(revision 76c37a99c13dbe456f0aa5adcf7439778e84a4c3)
+++ Source/Pythontest/TestUIOFiles.py	(revision 80281769fb906cbf4c8c64994964373c213ce779)
@@ -1,3 +1,3 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
@@ -51,8 +51,8 @@
     def test_str_(self):
         """Test if list of file is correctly converted to string."""
-        self.expected = "FCSH__SL.20160410.40429.16424.grb, "\
+        self.expected = "FCOG__ML.20160410.40429.16424.grb, "\
+                        "FCOG__SL.20160410.40429.16424.grb, "\
                         "FCSH__ML.20160410.40429.16424.grb, "\
-                        "FCOG__SL.20160410.40429.16424.grb, "\
-                        "FCOG__ML.20160410.40429.16424.grb, "\
+                        "FCSH__SL.20160410.40429.16424.grb, "\
                         "OG_OROLSM__SL.20160410.40429.16424.grb, "\
                         "FCGG__SL.20160410.40429.16424.grb, "\
Index: Source/Pythontest/_config_test.py
===================================================================
--- Source/Pythontest/_config_test.py	(revision 76c37a99c13dbe456f0aa5adcf7439778e84a4c3)
+++ Source/Pythontest/_config_test.py	(revision 80281769fb906cbf4c8c64994964373c213ce779)
@@ -10,5 +10,5 @@
 
  @License:
-    (C) Copyright 2014-2018.
+    (C) Copyright 2014-2020.
 
     This software is licensed under the terms of the Apache Licence Version 2.0
Index: Source/Pythontest/leos_test_suit.py
===================================================================
--- Source/Pythontest/leos_test_suit.py	(revision ba9923062b4af98cbdd20c4101c305766fc728f5)
+++ 	(revision )
@@ -1,104 +1,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#*******************************************************************************
-# @Author: Leopold Haimberger (University of Vienna)
-#
-# @Date: December 2015
-#
-# @Change History:
-#
-#    February 2018 - Anne Philipp (University of Vienna):
-#        - applied PEP8 style guide
-#        - added documentation
-#
-# @License:
-#    (C) Copyright 2015-2018.
-#
-#    This software is licensed under the terms of the Apache Licence Version 2.0
-#    which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# @Program Functionality:
-#    This script triggers the flex_extract test suite. Call with
-#    test_suite.py [test group]
-#
-# @Program Content:
-#
-#*******************************************************************************
-
-# ------------------------------------------------------------------------------
-# MODULES
-# ------------------------------------------------------------------------------
-import os
-import sys
-import json
-import subprocess
-
-# ------------------------------------------------------------------------------
-# PROGRAM
-# ------------------------------------------------------------------------------
-try:
-    taskfile = open('test_suite.json')
-except IOError:
-    print 'could not open suite definition file test_suite.json'
-    exit()
-
-if not os.path.isfile('../src/CONVERT2'):
-    print '../src/CONVERT2 could not be found'
-    print 'please run "install.py --target=local" first'
-    exit()
-
-fprs = os.getenv('FLEXPART_ROOT_SCRIPTS')
-if fprs is None:
-    print 'FLEXPART_ROOT_SCRIPTS not set .. some test jobs may fail'
-
-tasks = json.load(taskfile, encoding='latin-1')
-taskfile.close()
-if not os.path.exists('../test'):
-    os.makedirs('../test')
-if len(sys.argv) > 1:
-    groups = sys.argv[1:]
-else:
-    groups = ['xinstall', 'default', 'ops', 'work', 'cv', 'fc']#,'hires']
-jobcounter = 0
-jobfailed = 0
-for g in groups:
-    try:
-        tk, tv = g, tasks[g]
-    finally:
-        pass
-    garglist = []
-    for ttk, ttv in tv.iteritems():
-        if isinstance(ttv, basestring):
-            if ttk != 'script':
-                garglist.append('--' + ttk)
-                if ttv[0] == '$':
-                    garglist.append(os.path.expandvars(ttv))
-                else:
-                    garglist.append(ttv)
-    for ttk, ttv in tv.iteritems():
-        if isinstance(ttv, dict):
-            arglist = []
-            for tttk, tttv in ttv.iteritems():
-                if isinstance(tttv, basestring):
-                    arglist.append('--' + tttk)
-                    if '$' in tttv[0]:
-                        arglist.append(os.path.expandvars(tttv))
-                    else:
-                        arglist.append(tttv)
-            print 'Command: ', ' '.join([tv['script']] + garglist + arglist)
-            o = '../test/' + tk + '_' + ttk + '_' + '_'.join(ttv.keys())
-            print 'Output will be sent to ', o
-            f = open(o, 'w')
-            try:
-                p = subprocess.check_call([tv['script']] + garglist + arglist,
-                                          stdout=f, stderr=f)
-            except subprocess.CalledProcessError as e:
-                f.write('\nFAILED\n')
-                print 'FAILED'
-                jobfailed += 1
-            jobcounter += 1
-            f.close()
-
-print 'Test suite tasks completed'
-print str(jobcounter-jobfailed) + ' successful, ' + str(jobfailed) + ' failed'
-print 'If tasks have been submitted via ECACCESS please check emails'
Index: Source/Pythontest/leos_testsuite.json
===================================================================
--- Source/Pythontest/leos_testsuite.json	(revision ba9923062b4af98cbdd20c4101c305766fc728f5)
+++ 	(revision )
@@ -1,79 +1,0 @@
-{
-"install": {
-"script": "install.py",
-"control": null,
-"inputdir": null,
-"ecuid":"lh0",
-"ecgid":"spatlh00",
-"gateway":"srvx7.img.univie.ac.at",
-"destination":"leo@genericSftp",
-"local": {"target":"local"},
-"local_flexpart": {"target":"local","flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"},
-"ecgate": {"target":"ecgate"},
-"cca": {"target":"cca"}
-},
-"default": {
-"script": "submit.py",
-"control": null,
-"inputdir": null,
-"start_date": "20131107",
-"local": {},
-"local_flexpart": {"flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"},
-"ecgate": {"queue":"ecgate"},
-"cca": {"queue":"cca"}
-},
-"work": {
-"script": "submit.py",
-"control": null,
-"start_date": "20131107",
-"inputdir": "$SCRATCH/work",
-"local_flexpart": {"flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"}
-},
-"fc": {
-"script": "submit.py",
-"control": "CONTROL_FC",
-"inputdir": "$SCRATCH/workfc",
-"local_flexpart": {"flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"}
-},
-"cv": {
-"script": "submit.py",
-"control": "CONTROL_CV",
-"inputdir": "$SCRATCH/workcv",
-"start_date": "20131107",
-"local_flexpart": {"flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"}
-},
-"hires": {
-"script": "submit.py",
-"control": "CONTROL_HIRES",
-"inputdir": "$SCRATCH/workhires",
-"local_flexpart": {"flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"},
-"ecgate": {"queue":"ecgate"}
-},
-"hiresgauss": {
-"script": "submit.py",
-"control": "CONTROL_HIRESGAUSS",
-"inputdir": "$SCRATCH/workhiresgauss",
-"basetime": "00",
-"local_flexpart": {"flexpart_root_scripts":"$FLEXPART_ROOT_SCRIPTS"},
-"ecgate": {"queue":"ecgate"},
-"cca": {"queue":"cca"}
-},
-"ops": {
-"script": "submit.py",
-"start_date": "20131108",
-"control": "CONTROL_OPS_V6.0_4V.temp",
-"inputdir": "$SCRATCH/workops2",
-"local_flexpart": {"basetime":"00"},
-"local_flexpart12": {"basetime":"12"},
-"ecgate": {"queue":"ecgate","basetime":"00"}
-},
-"opsfc": {
-"script": "submit.py",
-"start_date": "20131108",
-"control": "CONTROL_OPS_V6.0",
-"inputdir": "$SCRATCH/workopsfc",
-"local_flexpart": {"basetime":"00"},
-"local_flexpart12": {"basetime":"12"},
-"ecgate": {"queue":"ecgate","basetime":"00"}
-}
-}
