Changeset b6ea29e in flex_extract.git

Timestamp:

Jul 28, 2022, 10:34:17 AM (21 months ago)

Author:

Anne Tipka <anne.tipka@…>

Branches:

dev

Children:

373b74e

Parents:

5606fc9

Message:

Adaptations for Bologna servers in Python part

Location:

Source/Python

Files:

• Mods/tools.py (modified) (3 diffs)
• _config.py (modified) (3 diffs)
• install.py (modified) (7 diffs)

Source/Python/Mods/tools.py

@@ -651 +651 @@
     return
 
-def put_file_to_ecserver(ecd, filename, target, ecuid, ecgid):
+def put_file_to_ecserver(ecd, filename):
     '''Uses the ecaccess-file-put command to send a file to the ECMWF servers.
 
@@ -668 +668 @@
         The name of the file to send to the ECMWF server.
 
-    target : str
-        The target queue where the file should be sent to.
-
-    ecuid : str
-        The user id on ECMWF server.
-
-    ecgid : str
-        The group id on ECMWF server.
-
     Return
     ------
@@ -684 +675 @@
     try:
         subprocess.check_output(['ecaccess-file-put',
-                                 ecd + '/' + filename,
-                                 target + ':/home/ms/' +
-                                 ecgid + '/' + ecuid +
-                                 '/' + filename],
+                                 ecd + '/' + filename],
                                 stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as e:

Source/Python/_config.py

@@ -61 +61 @@
     ec_hostname = os.getenv(HOSTENV_BOLOGNA)
     FLAG_ON_ECMWFSERVER = ec_hostname in HOSTNAMES_BOLOGNA_LIST
-    QUEUES_LIST = ['ecs', 'ecs-login', 'hpc', 'hpc-login', 'hpc-2020']
+    QUEUES_LIST = ['ecs', 'hpc']
 # Test and set ECMWF Reading server values
 elif os.getenv(HOSTENV_READING) is not None:
@@ -70 +70 @@
     FLAG_ON_ECMWFSERVER = False
 
-QUEUES_LIST_ALL = ['ecs', 'ecs-login', 'hpc', 'hpc-login', 'hpc-2020', 'ecgate', 'cca', 'ccb']
+QUEUES_LIST_ALL = ['ecs', 'hpc', 'ecgate', 'cca', 'ccb']
 
-INSTALL_TARGETS = ['local', 'syslocal', 'ecgate', 'cca', 'ccb']
+INSTALL_TARGETS = ['local', 'syslocal', 'ecgate', 'cca', 'ccb', 'ecs', 'hpc']
 
 CDS_DATASET_ML = 'reanalysis-era5-complete'
@@ -88 +88 @@
 TEMPFILE_USER_ENVVARS = 'ECMWF_ENV.template'
 FILE_USER_ENVVARS = 'ECMWF_ENV'
-TEMPFILE_INSTALL_COMPILEJOB = 'installscript.template'
+TEMPFILE_INSTALL_COMPILEJOB = 'installscript.template.bologna'
 FILE_INSTALL_COMPILEJOB = 'compilejob.ksh'
-TEMPFILE_INSTALL_JOB = 'jobscript.template'
+TEMPFILE_INSTALL_JOB = 'jobscript.template.bologna'
 TEMPFILE_JOB = 'submitscript.template'
 FILE_JOB_OD = 'job.ksh'

Source/Python/install.py

@@ -110 +110 @@
     check_install_conditions(c)
 
-    if c.install_target.lower() not in ['local', 'syslocal']: # ecgate or cca
+    if c.install_target.lower() not in ['local', 'syslocal']:  # ecmwf servers e.g. ecgate (ecs) and hpc
         install_via_gateway(c)
     else: # local
@@ -137 +137 @@
     parser.add_argument('--target', dest='install_target',
                         type=none_or_str, default=None,
-                        help="Valid targets: syslocal | local | ecgate | cca , \
-                        the latter two are at ECMWF")
+                        help="Valid targets: syslocal | local | ecgate | cca | \
+                        ccb | ecs | hpc , the latter 5 are at ECMWF servers")
     parser.add_argument("--makefile", dest="makefile",
                         type=none_or_str, default=None,
@@ -199 +199 @@
     tar_file = os.path.join(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
 
-    mk_compilejob(c.makefile, c.ecuid, c.ecgid, c.installdir)
+    mk_compilejob(c.makefile, c.ecuid, c.installdir)
 
     mk_job_template(c.ecuid, c.ecgid, c.installdir)
@@ -207 +207 @@
     mk_tarball(tar_file, c.install_target)
 
-    put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name,
-                         c.install_target, c.ecuid, c.ecgid)
+    put_file_to_ecserver(_config.PATH_FLEXEXTRACT_DIR, tarball_name)
 
     submit_job_to_ecserver(c.install_target,
@@ -614 +613 @@
     return
 
-def mk_compilejob(makefile, ecuid, ecgid, fp_root):
+def mk_compilejob(makefile, ecuid, fp_root):
     '''Modifies the original job template file so that it is specified
     for the user and the environment were it will be applied. Result
-    is stored in a new file "job.temp" in the python directory.
+    is stored in a new file "compilejob.ksh" in the Jobscript directory.
 
     Parameters
@@ -627 +626 @@
     ecuid : str
         The user id on ECMWF server.
-
-    ecgid : str
-        The group id on ECMWF server.
 
     fp_root : str
@@ -652 +648 @@
 
         stream = compile_template.generate(
-            usergroup=ecgid,
             username=ecuid,
             version_number=_config._VERSION_STR,