[274f9ef] | 1 | |
---|
| 2 | |
---|
| 3 | <!DOCTYPE html> |
---|
| 4 | <!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> |
---|
| 5 | <!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> |
---|
| 6 | <head> |
---|
| 7 | <meta charset="utf-8"> |
---|
| 8 | |
---|
| 9 | <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
---|
| 10 | |
---|
| 11 | <title>install — flex_extract 7.1 alpha documentation</title> |
---|
| 12 | |
---|
| 13 | |
---|
| 14 | |
---|
| 15 | |
---|
| 16 | |
---|
| 17 | |
---|
| 18 | |
---|
| 19 | |
---|
| 20 | |
---|
| 21 | |
---|
| 22 | |
---|
| 23 | |
---|
| 24 | |
---|
| 25 | |
---|
| 26 | |
---|
| 27 | <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> |
---|
| 28 | <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> |
---|
| 29 | <link rel="index" title="Index" href="../genindex.html" /> |
---|
| 30 | <link rel="search" title="Search" href="../search.html" /> |
---|
| 31 | |
---|
| 32 | |
---|
| 33 | <script src="../_static/js/modernizr.min.js"></script> |
---|
| 34 | |
---|
| 35 | </head> |
---|
| 36 | |
---|
| 37 | <body class="wy-body-for-nav"> |
---|
| 38 | |
---|
| 39 | |
---|
| 40 | <div class="wy-grid-for-nav"> |
---|
| 41 | |
---|
| 42 | |
---|
| 43 | <nav data-toggle="wy-nav-shift" class="wy-nav-side"> |
---|
| 44 | <div class="wy-side-scroll"> |
---|
| 45 | <div class="wy-side-nav-search"> |
---|
| 46 | |
---|
| 47 | |
---|
| 48 | |
---|
| 49 | <a href="../index.html" class="icon icon-home"> flex_extract |
---|
| 50 | |
---|
| 51 | |
---|
| 52 | |
---|
| 53 | </a> |
---|
| 54 | |
---|
| 55 | |
---|
| 56 | |
---|
| 57 | |
---|
| 58 | <div class="version"> |
---|
| 59 | 7.1 |
---|
| 60 | </div> |
---|
| 61 | |
---|
| 62 | |
---|
| 63 | |
---|
| 64 | |
---|
| 65 | <div role="search"> |
---|
| 66 | <form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> |
---|
| 67 | <input type="text" name="q" placeholder="Search docs" /> |
---|
| 68 | <input type="hidden" name="check_keywords" value="yes" /> |
---|
| 69 | <input type="hidden" name="area" value="default" /> |
---|
| 70 | </form> |
---|
| 71 | </div> |
---|
| 72 | |
---|
| 73 | |
---|
| 74 | </div> |
---|
| 75 | |
---|
| 76 | <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> |
---|
| 77 | |
---|
| 78 | |
---|
| 79 | |
---|
| 80 | |
---|
| 81 | |
---|
| 82 | |
---|
| 83 | <p class="caption"><span class="caption-text">Table of Contents:</span></p> |
---|
| 84 | <ul> |
---|
| 85 | <li class="toctree-l1"><a class="reference internal" href="../downloads.html">Downloads</a></li> |
---|
| 86 | <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> |
---|
| 87 | <li class="toctree-l1"><a class="reference internal" href="../program_structure.html">Program Structure</a></li> |
---|
| 88 | <li class="toctree-l1"><a class="reference internal" href="../user_guide.html">User Guide</a></li> |
---|
| 89 | <li class="toctree-l1"><a class="reference internal" href="../api.html">Auto Generated Documentation</a></li> |
---|
| 90 | <li class="toctree-l1"><a class="reference internal" href="../support.html">Support</a></li> |
---|
| 91 | </ul> |
---|
| 92 | |
---|
| 93 | |
---|
| 94 | |
---|
| 95 | </div> |
---|
| 96 | </div> |
---|
| 97 | </nav> |
---|
| 98 | |
---|
| 99 | <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> |
---|
| 100 | |
---|
| 101 | |
---|
| 102 | <nav class="wy-nav-top" aria-label="top navigation"> |
---|
| 103 | |
---|
| 104 | <i data-toggle="wy-nav-top" class="fa fa-bars"></i> |
---|
| 105 | <a href="../index.html">flex_extract</a> |
---|
| 106 | |
---|
| 107 | </nav> |
---|
| 108 | |
---|
| 109 | |
---|
| 110 | <div class="wy-nav-content"> |
---|
| 111 | |
---|
| 112 | <div class="rst-content"> |
---|
| 113 | |
---|
| 114 | |
---|
| 115 | |
---|
| 116 | |
---|
| 117 | |
---|
| 118 | |
---|
| 119 | |
---|
| 120 | |
---|
| 121 | |
---|
| 122 | |
---|
| 123 | |
---|
| 124 | |
---|
| 125 | |
---|
| 126 | |
---|
| 127 | |
---|
| 128 | |
---|
| 129 | |
---|
| 130 | <div role="navigation" aria-label="breadcrumbs navigation"> |
---|
| 131 | |
---|
| 132 | <ul class="wy-breadcrumbs"> |
---|
| 133 | |
---|
| 134 | <li><a href="../index.html">Docs</a> »</li> |
---|
| 135 | |
---|
| 136 | <li><a href="index.html">Module code</a> »</li> |
---|
| 137 | |
---|
| 138 | <li>install</li> |
---|
| 139 | |
---|
| 140 | |
---|
| 141 | <li class="wy-breadcrumbs-aside"> |
---|
| 142 | |
---|
| 143 | </li> |
---|
| 144 | |
---|
| 145 | </ul> |
---|
| 146 | |
---|
| 147 | |
---|
| 148 | <hr/> |
---|
| 149 | </div> |
---|
| 150 | <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> |
---|
| 151 | <div itemprop="articleBody"> |
---|
| 152 | |
---|
| 153 | <h1>Source code for install</h1><div class="highlight"><pre> |
---|
| 154 | <span></span><span class="ch">#!/usr/bin/env python</span> |
---|
| 155 | <span class="c1"># -*- coding: utf-8 -*-</span> |
---|
| 156 | <span class="c1">#*******************************************************************************</span> |
---|
| 157 | <span class="c1"># @Author: Leopold Haimberger (University of Vienna)</span> |
---|
| 158 | <span class="c1">#</span> |
---|
| 159 | <span class="c1"># @Date: November 2015</span> |
---|
| 160 | <span class="c1">#</span> |
---|
| 161 | <span class="c1"># @Change History:</span> |
---|
| 162 | <span class="c1">#</span> |
---|
| 163 | <span class="c1"># February 2018 - Anne Philipp (University of Vienna):</span> |
---|
| 164 | <span class="c1"># - applied PEP8 style guide</span> |
---|
| 165 | <span class="c1"># - added documentation</span> |
---|
| 166 | <span class="c1"># - moved install_args_and_control in here</span> |
---|
| 167 | <span class="c1"># - splitted code in smaller functions</span> |
---|
| 168 | <span class="c1"># - delete convert build files in here instead of compile job script</span> |
---|
| 169 | <span class="c1"># - changed static path names to Variables from config file</span> |
---|
| 170 | <span class="c1">#</span> |
---|
| 171 | <span class="c1"># @License:</span> |
---|
| 172 | <span class="c1"># (C) Copyright 2015-2018.</span> |
---|
| 173 | <span class="c1">#</span> |
---|
| 174 | <span class="c1"># This software is licensed under the terms of the Apache Licence Version 2.0</span> |
---|
| 175 | <span class="c1"># which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.</span> |
---|
| 176 | <span class="c1">#</span> |
---|
| 177 | <span class="c1"># @Program Functionality:</span> |
---|
| 178 | <span class="c1"># Depending on the selected installation environment (locally or on the</span> |
---|
| 179 | <span class="c1"># ECMWF server ecgate or cca) the program extracts the commandline</span> |
---|
| 180 | <span class="c1"># arguments and the CONTROL file parameter and prepares the corresponding</span> |
---|
| 181 | <span class="c1"># environment. The necessary files are collected in a tar-ball and placed</span> |
---|
| 182 | <span class="c1"># at the target location. There its untared, the environment variables will</span> |
---|
| 183 | <span class="c1"># be set and the Fortran code will be compiled. If the ECMWF environment is</span> |
---|
| 184 | <span class="c1"># selected a job script is prepared and submitted for the remaining</span> |
---|
| 185 | <span class="c1"># configurations after putting the tar-ball to the target ECMWF server.</span> |
---|
| 186 | <span class="c1">#</span> |
---|
| 187 | <span class="c1"># @Program Content:</span> |
---|
| 188 | <span class="c1"># - main</span> |
---|
| 189 | <span class="c1"># - get_install_cmdline_arguments</span> |
---|
| 190 | <span class="c1"># - install_via_gateway</span> |
---|
| 191 | <span class="c1"># - mk_tarball</span> |
---|
| 192 | <span class="c1"># - un_tarball</span> |
---|
| 193 | <span class="c1"># - mk_env_vars</span> |
---|
| 194 | <span class="c1"># - mk_compilejob</span> |
---|
| 195 | <span class="c1"># - mk_job_template</span> |
---|
| 196 | <span class="c1"># - delete_convert_build</span> |
---|
| 197 | <span class="c1"># - make_convert_build</span> |
---|
| 198 | <span class="c1">#</span> |
---|
| 199 | <span class="c1">#*******************************************************************************</span> |
---|
| 200 | |
---|
| 201 | <span class="c1"># ------------------------------------------------------------------------------</span> |
---|
| 202 | <span class="c1"># MODULES</span> |
---|
| 203 | <span class="c1"># ------------------------------------------------------------------------------</span> |
---|
| 204 | <span class="kn">import</span> <span class="nn">os</span> |
---|
| 205 | <span class="kn">import</span> <span class="nn">sys</span> |
---|
| 206 | <span class="kn">import</span> <span class="nn">glob</span> |
---|
| 207 | <span class="kn">import</span> <span class="nn">subprocess</span> |
---|
| 208 | <span class="kn">import</span> <span class="nn">inspect</span> |
---|
| 209 | <span class="kn">from</span> <span class="nn">argparse</span> <span class="k">import</span> <span class="n">ArgumentParser</span><span class="p">,</span> <span class="n">ArgumentDefaultsHelpFormatter</span> |
---|
| 210 | |
---|
| 211 | <span class="c1"># software specific classes and modules from flex_extract</span> |
---|
| 212 | <span class="kn">import</span> <span class="nn">_config</span> |
---|
| 213 | <span class="kn">from</span> <span class="nn">classes.ControlFile</span> <span class="k">import</span> <span class="n">ControlFile</span> |
---|
| 214 | <span class="kn">from</span> <span class="nn">classes.UioFiles</span> <span class="k">import</span> <span class="n">UioFiles</span> |
---|
| 215 | <span class="kn">from</span> <span class="nn">mods.tools</span> <span class="k">import</span> <span class="n">make_dir</span><span class="p">,</span> <span class="n">put_file_to_ecserver</span><span class="p">,</span> <span class="n">submit_job_to_ecserver</span> |
---|
| 216 | |
---|
| 217 | <span class="c1"># ------------------------------------------------------------------------------</span> |
---|
| 218 | <span class="c1"># FUNCTIONS</span> |
---|
| 219 | <span class="c1"># ------------------------------------------------------------------------------</span> |
---|
| 220 | <div class="viewcode-block" id="main"><a class="viewcode-back" href="../api.html#install.main">[docs]</a><span class="k">def</span> <span class="nf">main</span><span class="p">():</span> |
---|
| 221 | <span class="sd">'''Controls the installation process. Calls the installation function</span> |
---|
| 222 | <span class="sd"> if target is specified.</span> |
---|
| 223 | |
---|
| 224 | <span class="sd"> Parameters</span> |
---|
| 225 | <span class="sd"> ----------</span> |
---|
| 226 | |
---|
| 227 | <span class="sd"> Return</span> |
---|
| 228 | <span class="sd"> ------</span> |
---|
| 229 | <span class="sd"> '''</span> |
---|
| 230 | |
---|
| 231 | <span class="n">args</span> <span class="o">=</span> <span class="n">get_install_cmdline_arguments</span><span class="p">()</span> |
---|
| 232 | <span class="n">c</span> <span class="o">=</span> <span class="n">ControlFile</span><span class="p">(</span><span class="n">args</span><span class="o">.</span><span class="n">controlfile</span><span class="p">)</span> |
---|
| 233 | <span class="n">c</span><span class="o">.</span><span class="n">assign_args_to_control</span><span class="p">(</span><span class="n">args</span><span class="p">)</span> |
---|
| 234 | <span class="n">c</span><span class="o">.</span><span class="n">check_install_conditions</span><span class="p">()</span> |
---|
| 235 | |
---|
| 236 | <span class="n">install_via_gateway</span><span class="p">(</span><span class="n">c</span><span class="p">)</span> |
---|
| 237 | |
---|
| 238 | <span class="k">return</span></div> |
---|
| 239 | |
---|
| 240 | <div class="viewcode-block" id="get_install_cmdline_arguments"><a class="viewcode-back" href="../api.html#install.get_install_cmdline_arguments">[docs]</a><span class="k">def</span> <span class="nf">get_install_cmdline_arguments</span><span class="p">():</span> |
---|
| 241 | <span class="sd">'''Decomposes the command line arguments and assigns them to variables.</span> |
---|
| 242 | <span class="sd"> Apply default values for non mentioned arguments.</span> |
---|
| 243 | |
---|
| 244 | <span class="sd"> Parameters</span> |
---|
| 245 | <span class="sd"> ----------</span> |
---|
| 246 | |
---|
| 247 | <span class="sd"> Return</span> |
---|
| 248 | <span class="sd"> ------</span> |
---|
| 249 | <span class="sd"> args : :obj:`Namespace`</span> |
---|
| 250 | <span class="sd"> Contains the commandline arguments from script/program call.</span> |
---|
| 251 | <span class="sd"> '''</span> |
---|
| 252 | <span class="n">parser</span> <span class="o">=</span> <span class="n">ArgumentParser</span><span class="p">(</span><span class="n">description</span><span class="o">=</span><span class="s1">'Install flex_extract software locally or </span><span class="se">\</span> |
---|
| 253 | <span class="s1"> on ECMWF machines'</span><span class="p">,</span> |
---|
| 254 | <span class="n">formatter_class</span><span class="o">=</span><span class="n">ArgumentDefaultsHelpFormatter</span><span class="p">)</span> |
---|
| 255 | |
---|
| 256 | <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s1">'--target'</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="s1">'install_target'</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
---|
| 257 | <span class="n">help</span><span class="o">=</span><span class="s2">"Valid targets: local | ecgate | cca , </span><span class="se">\</span> |
---|
| 258 | <span class="s2"> the latter two are at ECMWF"</span><span class="p">)</span> |
---|
| 259 | <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"--makefile"</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="s2">"makefile"</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
---|
| 260 | <span class="n">help</span><span class="o">=</span><span class="s1">'Name of Makefile to use for compiling CONVERT2'</span><span class="p">)</span> |
---|
| 261 | <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"--ecuid"</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="s2">"ecuid"</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
---|
| 262 | <span class="n">help</span><span class="o">=</span><span class="s1">'user id at ECMWF'</span><span class="p">)</span> |
---|
| 263 | <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"--ecgid"</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="s2">"ecgid"</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
---|
| 264 | <span class="n">help</span><span class="o">=</span><span class="s1">'group id at ECMWF'</span><span class="p">)</span> |
---|
| 265 | <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"--gateway"</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="s2">"gateway"</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
---|
| 266 | <span class="n">help</span><span class="o">=</span><span class="s1">'name of local gateway server'</span><span class="p">)</span> |
---|
| 267 | <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"--destination"</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="s2">"destination"</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> |
---|
| 268 | <span class="n">help</span><span class="o">=</span><span class="s1">'ecaccess destination, e.g. leo@genericSftp'</span><span class="p">)</span> |
---|
| 269 | |
---|
| 270 | <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"--flexpart_root_scripts"</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="s2">"flexpart_root_scripts"</span><span class="p">,</span> |
---|
| 271 | <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">help</span><span class="o">=</span><span class="s2">"FLEXPART root directory on ECMWF </span><span class="se">\</span> |
---|
| 272 | <span class="s2"> servers (to find grib2flexpart and COMMAND file)</span><span class="se">\n\</span> |
---|
| 273 | <span class="s2"> Normally flex_extract resides in the scripts directory </span><span class="se">\</span> |
---|
| 274 | <span class="s2"> of the FLEXPART distribution."</span><span class="p">)</span> |
---|
| 275 | |
---|
| 276 | <span class="c1"># arguments for job submission to ECMWF, only needed by submit.py</span> |
---|
| 277 | <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"--job_template"</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="s1">'job_template'</span><span class="p">,</span> |
---|
| 278 | <span class="n">default</span><span class="o">=</span><span class="s2">"job.temp.o"</span><span class="p">,</span> |
---|
| 279 | <span class="n">help</span><span class="o">=</span><span class="s2">"job template file for submission to ECMWF"</span><span class="p">)</span> |
---|
| 280 | |
---|
| 281 | <span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"--controlfile"</span><span class="p">,</span> <span class="n">dest</span><span class="o">=</span><span class="s2">"controlfile"</span><span class="p">,</span> |
---|
| 282 | <span class="n">default</span><span class="o">=</span><span class="s1">'CONTROL.temp'</span><span class="p">,</span> |
---|
| 283 | <span class="n">help</span><span class="o">=</span><span class="s2">"file with CONTROL parameters"</span><span class="p">)</span> |
---|
| 284 | |
---|
| 285 | <span class="n">args</span> <span class="o">=</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse_args</span><span class="p">()</span> |
---|
| 286 | |
---|
| 287 | <span class="k">return</span> <span class="n">args</span></div> |
---|
| 288 | |
---|
| 289 | |
---|
| 290 | <div class="viewcode-block" id="install_via_gateway"><a class="viewcode-back" href="../api.html#install.install_via_gateway">[docs]</a><span class="k">def</span> <span class="nf">install_via_gateway</span><span class="p">(</span><span class="n">c</span><span class="p">):</span> |
---|
| 291 | <span class="sd">'''Perform the actual installation on local machine or prepare data</span> |
---|
| 292 | <span class="sd"> transfer to remote gate and submit a job script which will</span> |
---|
| 293 | <span class="sd"> install everything on the remote gate.</span> |
---|
| 294 | |
---|
| 295 | <span class="sd"> Parameters</span> |
---|
| 296 | <span class="sd"> ----------</span> |
---|
| 297 | <span class="sd"> c : :obj:`ControlFile`</span> |
---|
| 298 | <span class="sd"> Contains all the parameters of CONTROL file and</span> |
---|
| 299 | <span class="sd"> command line.</span> |
---|
| 300 | |
---|
| 301 | <span class="sd"> Return</span> |
---|
| 302 | <span class="sd"> ------</span> |
---|
| 303 | |
---|
| 304 | <span class="sd"> '''</span> |
---|
| 305 | <span class="kn">import</span> <span class="nn">tarfile</span> |
---|
| 306 | |
---|
| 307 | <span class="n">ecd</span> <span class="o">=</span> <span class="n">_config</span><span class="o">.</span><span class="n">PATH_FLEXEXTRACT_DIR</span> |
---|
| 308 | <span class="n">tarball_name</span> <span class="o">=</span> <span class="n">_config</span><span class="o">.</span><span class="n">FLEXEXTRACT_DIRNAME</span> <span class="o">+</span> <span class="s1">'.tar'</span> |
---|
| 309 | <span class="n">tar_file</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">ecd</span><span class="p">,</span> <span class="n">tarball_name</span><span class="p">)</span> |
---|
| 310 | |
---|
| 311 | <span class="n">target_dirname</span> <span class="o">=</span> <span class="n">_config</span><span class="o">.</span><span class="n">FLEXEXTRACT_DIRNAME</span> |
---|
| 312 | <span class="n">fortran_executable</span> <span class="o">=</span> <span class="n">_config</span><span class="o">.</span><span class="n">FORTRAN_EXECUTABLE</span> |
---|
| 313 | |
---|
| 314 | <span class="k">if</span> <span class="n">c</span><span class="o">.</span><span class="n">install_target</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">!=</span> <span class="s1">'local'</span><span class="p">:</span> <span class="c1"># ecgate or cca</span> |
---|
| 315 | |
---|
| 316 | <span class="n">mk_compilejob</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">makefile</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">install_target</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">ecuid</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">ecgid</span><span class="p">,</span> |
---|
| 317 | <span class="n">c</span><span class="o">.</span><span class="n">flexpart_root_scripts</span><span class="p">)</span> |
---|
| 318 | |
---|
| 319 | <span class="n">mk_job_template</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">ecuid</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">ecgid</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">gateway</span><span class="p">,</span> |
---|
| 320 | <span class="n">c</span><span class="o">.</span><span class="n">destination</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">flexpart_root_scripts</span><span class="p">)</span> |
---|
| 321 | |
---|
| 322 | <span class="n">mk_env_vars</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">ecuid</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">ecgid</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">gateway</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">destination</span><span class="p">)</span> |
---|
| 323 | |
---|
| 324 | <span class="n">mk_tarball</span><span class="p">(</span><span class="n">tar_file</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">install_target</span><span class="p">)</span> |
---|
| 325 | |
---|
| 326 | <span class="n">put_file_to_ecserver</span><span class="p">(</span><span class="n">ecd</span><span class="p">,</span> <span class="n">tarball_name</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">install_target</span><span class="p">,</span> |
---|
| 327 | <span class="n">c</span><span class="o">.</span><span class="n">ecuid</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">ecgid</span><span class="p">)</span> |
---|
| 328 | |
---|
| 329 | <span class="n">submit_job_to_ecserver</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">install_target</span><span class="p">,</span> |
---|
| 330 | <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_REL_JOBSCRIPTS</span><span class="p">,</span> |
---|
| 331 | <span class="n">_config</span><span class="o">.</span><span class="n">FILE_INSTALL_COMPILEJOB</span><span class="p">))</span> |
---|
| 332 | |
---|
| 333 | <span class="nb">print</span><span class="p">(</span><span class="s1">'job compilation script has been submitted to ecgate for '</span> <span class="o">+</span> |
---|
| 334 | <span class="s1">'installation in '</span> <span class="o">+</span> <span class="n">c</span><span class="o">.</span><span class="n">flexpart_root_scripts</span> <span class="o">+</span> |
---|
| 335 | <span class="s1">'/'</span> <span class="o">+</span> <span class="n">target_dirname</span><span class="p">)</span> |
---|
| 336 | <span class="nb">print</span><span class="p">(</span><span class="s1">'You should get an email with subject "flexcompile" within '</span> <span class="o">+</span> |
---|
| 337 | <span class="s1">'the next few minutes!'</span><span class="p">)</span> |
---|
| 338 | |
---|
| 339 | <span class="k">else</span><span class="p">:</span> <span class="c1">#local</span> |
---|
| 340 | <span class="k">if</span> <span class="n">c</span><span class="o">.</span><span class="n">flexpart_root_scripts</span> <span class="o">==</span> <span class="n">_config</span><span class="o">.</span><span class="n">PATH_FLEXEXTRACT_DIR</span> <span class="p">:</span> |
---|
| 341 | <span class="nb">print</span><span class="p">(</span><span class="s1">'WARNING: FLEXPART_ROOT_SCRIPTS has not been specified'</span><span class="p">)</span> |
---|
| 342 | <span class="nb">print</span><span class="p">(</span><span class="s1">'flex_extract will be installed in here by compiling the '</span> <span class="o">+</span> |
---|
| 343 | <span class="s1">'Fortran source in '</span> <span class="o">+</span> <span class="n">_config</span><span class="o">.</span><span class="n">PATH_FORTRAN_SRC</span><span class="p">)</span> |
---|
| 344 | <span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_FORTRAN_SRC</span><span class="p">)</span> |
---|
| 345 | <span class="k">else</span><span class="p">:</span> <span class="c1"># creates the target working directory for flex_extract</span> |
---|
| 346 | <span class="n">c</span><span class="o">.</span><span class="n">flexpart_root_scripts</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expandvars</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span> |
---|
| 347 | <span class="n">c</span><span class="o">.</span><span class="n">flexpart_root_scripts</span><span class="p">))</span> |
---|
| 348 | <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">ecd</span><span class="p">)</span> <span class="o">!=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">flexpart_root_scripts</span><span class="p">):</span> |
---|
| 349 | <span class="n">mk_tarball</span><span class="p">(</span><span class="n">tar_file</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">install_target</span><span class="p">)</span> |
---|
| 350 | <span class="n">make_dir</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">flexpart_root_scripts</span><span class="p">,</span> |
---|
| 351 | <span class="n">target_dirname</span><span class="p">))</span> |
---|
| 352 | <span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">flexpart_root_scripts</span><span class="p">,</span> |
---|
| 353 | <span class="n">target_dirname</span><span class="p">))</span> |
---|
| 354 | <span class="n">un_tarball</span><span class="p">(</span><span class="n">tar_file</span><span class="p">)</span> |
---|
| 355 | <span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">flexpart_root_scripts</span><span class="p">,</span> |
---|
| 356 | <span class="n">target_dirname</span><span class="p">,</span> |
---|
| 357 | <span class="n">_config</span><span class="o">.</span><span class="n">PATH_REL_FORTRAN_SRC</span><span class="p">))</span> |
---|
| 358 | |
---|
| 359 | <span class="c1"># Create Fortran executable - CONVERT2</span> |
---|
| 360 | <span class="nb">print</span><span class="p">(</span><span class="s1">'Install '</span> <span class="o">+</span> <span class="n">target_dirname</span> <span class="o">+</span> <span class="s1">' software at '</span> <span class="o">+</span> |
---|
| 361 | <span class="n">c</span><span class="o">.</span><span class="n">install_target</span> <span class="o">+</span> <span class="s1">' in directory '</span> <span class="o">+</span> |
---|
| 362 | <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">flexpart_root_scripts</span><span class="p">)</span> <span class="o">+</span> <span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">)</span> |
---|
| 363 | |
---|
| 364 | <span class="n">delete_convert_build</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span> |
---|
| 365 | <span class="n">make_convert_build</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">makefile</span><span class="p">)</span> |
---|
| 366 | |
---|
| 367 | <span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">ecd</span><span class="p">)</span> |
---|
| 368 | <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">tar_file</span><span class="p">):</span> |
---|
| 369 | <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">tar_file</span><span class="p">)</span> |
---|
| 370 | |
---|
| 371 | <span class="k">return</span></div> |
---|
| 372 | |
---|
| 373 | <div class="viewcode-block" id="mk_tarball"><a class="viewcode-back" href="../api.html#install.mk_tarball">[docs]</a><span class="k">def</span> <span class="nf">mk_tarball</span><span class="p">(</span><span class="n">tarball_path</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span> |
---|
| 374 | <span class="sd">'''Creates a tarball with all necessary files which need to be sent to the</span> |
---|
| 375 | <span class="sd"> installation directory.</span> |
---|
| 376 | <span class="sd"> It does not matter if this is local or remote.</span> |
---|
| 377 | <span class="sd"> Collects all python files, the Fortran source and makefiles,</span> |
---|
| 378 | <span class="sd"> the ECMWF_ENV file, the CONTROL files as well as the</span> |
---|
| 379 | <span class="sd"> template files.</span> |
---|
| 380 | |
---|
| 381 | <span class="sd"> Parameters</span> |
---|
| 382 | <span class="sd"> ----------</span> |
---|
| 383 | <span class="sd"> tarball_path : :obj:`string`</span> |
---|
| 384 | <span class="sd"> The complete path to the tar file which will contain all</span> |
---|
| 385 | <span class="sd"> relevant data for flex_extract.</span> |
---|
| 386 | |
---|
| 387 | <span class="sd"> target : :obj:`string`</span> |
---|
| 388 | <span class="sd"> The queue where the job is submitted to.</span> |
---|
| 389 | |
---|
| 390 | <span class="sd"> Return</span> |
---|
| 391 | <span class="sd"> ------</span> |
---|
| 392 | |
---|
| 393 | <span class="sd"> '''</span> |
---|
| 394 | <span class="kn">import</span> <span class="nn">tarfile</span> |
---|
| 395 | <span class="kn">from</span> <span class="nn">glob</span> <span class="k">import</span> <span class="n">glob</span> |
---|
| 396 | |
---|
| 397 | <span class="nb">print</span><span class="p">(</span><span class="s1">'Create tarball ...'</span><span class="p">)</span> |
---|
| 398 | |
---|
| 399 | <span class="c1"># change to FLEXEXTRACT directory so that the tar can contain</span> |
---|
| 400 | <span class="c1"># relative pathes to the files and directories</span> |
---|
| 401 | <span class="n">ecd</span> <span class="o">=</span> <span class="n">_config</span><span class="o">.</span><span class="n">PATH_FLEXEXTRACT_DIR</span> <span class="o">+</span> <span class="s1">'/'</span> |
---|
| 402 | <span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">ecd</span><span class="p">)</span> |
---|
| 403 | |
---|
| 404 | <span class="c1"># get lists of the files to be added to the tar file</span> |
---|
| 405 | <span class="k">if</span> <span class="n">target</span> <span class="o">==</span> <span class="s1">'local'</span><span class="p">:</span> |
---|
| 406 | <span class="n">ECMWF_ENV_FILE</span> <span class="o">=</span> <span class="p">[]</span> |
---|
| 407 | <span class="k">else</span><span class="p">:</span> |
---|
| 408 | <span class="n">ECMWF_ENV_FILE</span> <span class="o">=</span> <span class="p">[</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_REL_ECMWF_ENV</span><span class="p">]</span> |
---|
| 409 | |
---|
| 410 | <span class="n">pyfiles</span> <span class="o">=</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">relpath</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">ecd</span><span class="p">)</span> |
---|
| 411 | <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">UioFiles</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_LOCAL_PYTHON</span><span class="p">,</span> <span class="s1">'*py'</span><span class="p">)</span><span class="o">.</span><span class="n">files</span><span class="p">]</span> |
---|
| 412 | <span class="n">controlfiles</span> <span class="o">=</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">relpath</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">ecd</span><span class="p">)</span> |
---|
| 413 | <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">UioFiles</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_CONTROLFILES</span><span class="p">,</span> |
---|
| 414 | <span class="s1">'CONTROL*'</span><span class="p">)</span><span class="o">.</span><span class="n">files</span><span class="p">]</span> |
---|
| 415 | <span class="n">tempfiles</span> <span class="o">=</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">relpath</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">ecd</span><span class="p">)</span> |
---|
| 416 | <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">UioFiles</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_TEMPLATES</span> <span class="p">,</span> <span class="s1">'*'</span><span class="p">)</span><span class="o">.</span><span class="n">files</span><span class="p">]</span> |
---|
| 417 | <span class="n">ffiles</span> <span class="o">=</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">relpath</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">ecd</span><span class="p">)</span> |
---|
| 418 | <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">UioFiles</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_FORTRAN_SRC</span><span class="p">,</span> <span class="s1">'*.f*'</span><span class="p">)</span><span class="o">.</span><span class="n">files</span><span class="p">]</span> |
---|
| 419 | <span class="n">hfiles</span> <span class="o">=</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">relpath</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">ecd</span><span class="p">)</span> |
---|
| 420 | <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">UioFiles</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_FORTRAN_SRC</span><span class="p">,</span> <span class="s1">'*.h'</span><span class="p">)</span><span class="o">.</span><span class="n">files</span><span class="p">]</span> |
---|
| 421 | <span class="n">makefiles</span> <span class="o">=</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">relpath</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">ecd</span><span class="p">)</span> |
---|
| 422 | <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">UioFiles</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_FORTRAN_SRC</span><span class="p">,</span> <span class="s1">'Makefile*'</span><span class="p">)</span><span class="o">.</span><span class="n">files</span><span class="p">]</span> |
---|
| 423 | |
---|
| 424 | <span class="c1"># concatenate single lists to one for a better looping</span> |
---|
| 425 | <span class="n">filelist</span> <span class="o">=</span> <span class="n">pyfiles</span> <span class="o">+</span> <span class="n">controlfiles</span> <span class="o">+</span> <span class="n">tempfiles</span> <span class="o">+</span> <span class="n">ffiles</span> <span class="o">+</span> <span class="n">hfiles</span> <span class="o">+</span> \ |
---|
| 426 | <span class="n">makefiles</span> <span class="o">+</span> <span class="n">ECMWF_ENV_FILE</span> |
---|
| 427 | |
---|
| 428 | <span class="c1"># create installation tar-file</span> |
---|
| 429 | <span class="k">try</span><span class="p">:</span> |
---|
| 430 | <span class="k">with</span> <span class="n">tarfile</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">tarball_path</span><span class="p">,</span> <span class="s2">"w:gz"</span><span class="p">)</span> <span class="k">as</span> <span class="n">tar_handle</span><span class="p">:</span> |
---|
| 431 | <span class="k">for</span> <span class="n">file</span> <span class="ow">in</span> <span class="n">filelist</span><span class="p">:</span> |
---|
| 432 | <span class="n">tar_handle</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">file</span><span class="p">)</span> |
---|
| 433 | |
---|
| 434 | <span class="k">except</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">CalledProcessError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
---|
| 435 | <span class="nb">print</span><span class="p">(</span><span class="s1">'... ERROR CODE:</span><span class="se">\n</span><span class="s1"> ... '</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">returncode</span><span class="p">))</span> |
---|
| 436 | <span class="nb">print</span><span class="p">(</span><span class="s1">'... ERROR MESSAGE:</span><span class="se">\n</span><span class="s1"> ... '</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span> |
---|
| 437 | |
---|
| 438 | <span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="s1">'... could not make installation tar ball!'</span><span class="p">)</span> |
---|
| 439 | |
---|
| 440 | <span class="k">return</span></div> |
---|
| 441 | |
---|
| 442 | |
---|
| 443 | <div class="viewcode-block" id="un_tarball"><a class="viewcode-back" href="../api.html#install.un_tarball">[docs]</a><span class="k">def</span> <span class="nf">un_tarball</span><span class="p">(</span><span class="n">tarball_path</span><span class="p">):</span> |
---|
| 444 | <span class="sd">'''Extracts the given tarball into current directory.</span> |
---|
| 445 | |
---|
| 446 | <span class="sd"> Parameters</span> |
---|
| 447 | <span class="sd"> ----------</span> |
---|
| 448 | <span class="sd"> tarball_path : :obj:`string`</span> |
---|
| 449 | <span class="sd"> The complete path to the tar file which will contain all</span> |
---|
| 450 | <span class="sd"> relevant data for flex_extract.</span> |
---|
| 451 | |
---|
| 452 | <span class="sd"> Return</span> |
---|
| 453 | <span class="sd"> ------</span> |
---|
| 454 | |
---|
| 455 | <span class="sd"> '''</span> |
---|
| 456 | <span class="kn">import</span> <span class="nn">tarfile</span> |
---|
| 457 | |
---|
| 458 | <span class="nb">print</span><span class="p">(</span><span class="s1">'Untar ...'</span><span class="p">)</span> |
---|
| 459 | |
---|
| 460 | <span class="k">with</span> <span class="n">tarfile</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">tarball_path</span><span class="p">)</span> <span class="k">as</span> <span class="n">tar_handle</span><span class="p">:</span> |
---|
| 461 | <span class="n">tar_handle</span><span class="o">.</span><span class="n">extractall</span><span class="p">()</span> |
---|
| 462 | |
---|
| 463 | <span class="k">return</span></div> |
---|
| 464 | |
---|
| 465 | <div class="viewcode-block" id="mk_env_vars"><a class="viewcode-back" href="../api.html#install.mk_env_vars">[docs]</a><span class="k">def</span> <span class="nf">mk_env_vars</span><span class="p">(</span><span class="n">ecuid</span><span class="p">,</span> <span class="n">ecgid</span><span class="p">,</span> <span class="n">gateway</span><span class="p">,</span> <span class="n">destination</span><span class="p">):</span> |
---|
| 466 | <span class="sd">'''Creates a file named ECMWF_ENV which contains the</span> |
---|
| 467 | <span class="sd"> necessary environmental variables at ECMWF servers.</span> |
---|
| 468 | <span class="sd"> It is based on the template ECMWF_ENV.template.</span> |
---|
| 469 | |
---|
| 470 | <span class="sd"> Parameters</span> |
---|
| 471 | <span class="sd"> ----------</span> |
---|
| 472 | <span class="sd"> ecuid : :obj:`string`</span> |
---|
| 473 | <span class="sd"> The user id on ECMWF server.</span> |
---|
| 474 | |
---|
| 475 | <span class="sd"> ecgid : :obj:`string`</span> |
---|
| 476 | <span class="sd"> The group id on ECMWF server.</span> |
---|
| 477 | |
---|
| 478 | <span class="sd"> gateway : :obj:`string`</span> |
---|
| 479 | <span class="sd"> The gateway server the user is using.</span> |
---|
| 480 | |
---|
| 481 | <span class="sd"> destination : :obj:`string`</span> |
---|
| 482 | <span class="sd"> The remote destination which is used to transfer files</span> |
---|
| 483 | <span class="sd"> from ECMWF server to local gateway server.</span> |
---|
| 484 | |
---|
| 485 | <span class="sd"> Return</span> |
---|
| 486 | <span class="sd"> ------</span> |
---|
| 487 | |
---|
| 488 | <span class="sd"> '''</span> |
---|
| 489 | <span class="kn">from</span> <span class="nn">genshi.template.text</span> <span class="k">import</span> <span class="n">NewTextTemplate</span> |
---|
| 490 | <span class="kn">from</span> <span class="nn">genshi.template</span> <span class="k">import</span> <span class="n">TemplateLoader</span> |
---|
| 491 | |
---|
| 492 | <span class="n">loader</span> <span class="o">=</span> <span class="n">TemplateLoader</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_TEMPLATES</span><span class="p">,</span> <span class="n">auto_reload</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> |
---|
| 493 | <span class="n">ecmwfvars_template</span> <span class="o">=</span> <span class="n">loader</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">TEMPFILE_USER_ENVVARS</span><span class="p">,</span> |
---|
| 494 | <span class="bp">cls</span><span class="o">=</span><span class="n">NewTextTemplate</span><span class="p">)</span> |
---|
| 495 | |
---|
| 496 | <span class="n">stream</span> <span class="o">=</span> <span class="n">ecmwfvars_template</span><span class="o">.</span><span class="n">generate</span><span class="p">(</span><span class="n">user_name</span> <span class="o">=</span> <span class="n">ecuid</span><span class="p">,</span> |
---|
| 497 | <span class="n">user_group</span> <span class="o">=</span> <span class="n">ecgid</span><span class="p">,</span> |
---|
| 498 | <span class="n">gateway_name</span> <span class="o">=</span> <span class="n">gateway</span><span class="p">,</span> |
---|
| 499 | <span class="n">destination_name</span> <span class="o">=</span> <span class="n">destination</span> |
---|
| 500 | <span class="p">)</span> |
---|
| 501 | |
---|
| 502 | <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_ECMWF_ENV</span><span class="p">,</span> <span class="s1">'w'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span> |
---|
| 503 | <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">stream</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="s1">'text'</span><span class="p">))</span> |
---|
| 504 | |
---|
| 505 | <span class="k">return</span></div> |
---|
| 506 | |
---|
| 507 | <div class="viewcode-block" id="mk_compilejob"><a class="viewcode-back" href="../api.html#install.mk_compilejob">[docs]</a><span class="k">def</span> <span class="nf">mk_compilejob</span><span class="p">(</span><span class="n">makefile</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">ecuid</span><span class="p">,</span> <span class="n">ecgid</span><span class="p">,</span> <span class="n">fp_root</span><span class="p">):</span> |
---|
| 508 | <span class="sd">'''Modifies the original job template file so that it is specified</span> |
---|
| 509 | <span class="sd"> for the user and the environment were it will be applied. Result</span> |
---|
| 510 | <span class="sd"> is stored in a new file "job.temp" in the python directory.</span> |
---|
| 511 | |
---|
| 512 | <span class="sd"> Parameters</span> |
---|
| 513 | <span class="sd"> ----------</span> |
---|
| 514 | <span class="sd"> makefile : :obj:`string`</span> |
---|
| 515 | <span class="sd"> Name of the makefile which should be used to compile FORTRAN</span> |
---|
| 516 | <span class="sd"> CONVERT2 program.</span> |
---|
| 517 | |
---|
| 518 | <span class="sd"> target : :obj:`string`</span> |
---|
| 519 | <span class="sd"> The target where the installation should be done, e.g. the queue.</span> |
---|
| 520 | |
---|
| 521 | <span class="sd"> ecuid : :obj:`string`</span> |
---|
| 522 | <span class="sd"> The user id on ECMWF server.</span> |
---|
| 523 | |
---|
| 524 | <span class="sd"> ecgid : :obj:`string`</span> |
---|
| 525 | <span class="sd"> The group id on ECMWF server.</span> |
---|
| 526 | |
---|
| 527 | <span class="sd"> fp_root : :obj:`string`</span> |
---|
| 528 | <span class="sd"> Path to the root directory of FLEXPART environment or flex_extract</span> |
---|
| 529 | <span class="sd"> environment.</span> |
---|
| 530 | |
---|
| 531 | <span class="sd"> Return</span> |
---|
| 532 | <span class="sd"> ------</span> |
---|
| 533 | |
---|
| 534 | <span class="sd"> '''</span> |
---|
| 535 | <span class="kn">from</span> <span class="nn">genshi.template.text</span> <span class="k">import</span> <span class="n">NewTextTemplate</span> |
---|
| 536 | <span class="kn">from</span> <span class="nn">genshi.template</span> <span class="k">import</span> <span class="n">TemplateLoader</span> |
---|
| 537 | |
---|
| 538 | <span class="n">loader</span> <span class="o">=</span> <span class="n">TemplateLoader</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_TEMPLATES</span><span class="p">,</span> <span class="n">auto_reload</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> |
---|
| 539 | <span class="n">compile_template</span> <span class="o">=</span> <span class="n">loader</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">TEMPFILE_INSTALL_COMPILEJOB</span><span class="p">,</span> |
---|
| 540 | <span class="bp">cls</span><span class="o">=</span><span class="n">NewTextTemplate</span><span class="p">)</span> |
---|
| 541 | |
---|
| 542 | <span class="k">if</span> <span class="n">fp_root</span> <span class="o">==</span> <span class="s1">'../'</span><span class="p">:</span> |
---|
| 543 | <span class="n">fp_root</span> <span class="o">=</span> <span class="s1">'$HOME'</span> |
---|
| 544 | |
---|
| 545 | <span class="n">stream</span> <span class="o">=</span> <span class="n">compile_template</span><span class="o">.</span><span class="n">generate</span><span class="p">(</span> |
---|
| 546 | <span class="n">usergroup</span> <span class="o">=</span> <span class="n">ecgid</span><span class="p">,</span> |
---|
| 547 | <span class="n">username</span> <span class="o">=</span> <span class="n">ecuid</span><span class="p">,</span> |
---|
| 548 | <span class="n">version_number</span> <span class="o">=</span> <span class="n">_config</span><span class="o">.</span><span class="n">_VERSION_STR</span><span class="p">,</span> |
---|
| 549 | <span class="n">fp_root_scripts</span> <span class="o">=</span> <span class="n">fp_root</span><span class="p">,</span> |
---|
| 550 | <span class="n">makefile</span> <span class="o">=</span> <span class="n">makefile</span><span class="p">,</span> |
---|
| 551 | <span class="n">fortran_program</span> <span class="o">=</span> <span class="n">_config</span><span class="o">.</span><span class="n">FORTRAN_EXECUTABLE</span> |
---|
| 552 | <span class="p">)</span> |
---|
| 553 | |
---|
| 554 | <span class="n">compilejob</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_JOBSCRIPTS</span><span class="p">,</span> |
---|
| 555 | <span class="n">_config</span><span class="o">.</span><span class="n">FILE_INSTALL_COMPILEJOB</span><span class="p">)</span> |
---|
| 556 | |
---|
| 557 | <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">compilejob</span><span class="p">,</span> <span class="s1">'w'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span> |
---|
| 558 | <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">stream</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="s1">'text'</span><span class="p">))</span> |
---|
| 559 | |
---|
| 560 | <span class="k">return</span></div> |
---|
| 561 | |
---|
| 562 | <div class="viewcode-block" id="mk_job_template"><a class="viewcode-back" href="../api.html#install.mk_job_template">[docs]</a><span class="k">def</span> <span class="nf">mk_job_template</span><span class="p">(</span><span class="n">ecuid</span><span class="p">,</span> <span class="n">ecgid</span><span class="p">,</span> <span class="n">gateway</span><span class="p">,</span> <span class="n">destination</span><span class="p">,</span> <span class="n">fp_root</span><span class="p">):</span> |
---|
| 563 | <span class="sd">'''Modifies the original job template file so that it is specified</span> |
---|
| 564 | <span class="sd"> for the user and the environment were it will be applied. Result</span> |
---|
| 565 | <span class="sd"> is stored in a new file.</span> |
---|
| 566 | |
---|
| 567 | <span class="sd"> Parameters</span> |
---|
| 568 | <span class="sd"> ----------</span> |
---|
| 569 | <span class="sd"> ecuid : :obj:`string`</span> |
---|
| 570 | <span class="sd"> The user id on ECMWF server.</span> |
---|
| 571 | |
---|
| 572 | <span class="sd"> ecgid : :obj:`string`</span> |
---|
| 573 | <span class="sd"> The group id on ECMWF server.</span> |
---|
| 574 | |
---|
| 575 | <span class="sd"> gateway : :obj:`string`</span> |
---|
| 576 | <span class="sd"> The gateway server the user is using.</span> |
---|
| 577 | |
---|
| 578 | <span class="sd"> destination : :obj:`string`</span> |
---|
| 579 | <span class="sd"> The remote destination which is used to transfer files</span> |
---|
| 580 | <span class="sd"> from ECMWF server to local gateway server.</span> |
---|
| 581 | |
---|
| 582 | <span class="sd"> fp_root : :obj:`string`</span> |
---|
| 583 | <span class="sd"> Path to the root directory of FLEXPART environment or flex_extract</span> |
---|
| 584 | <span class="sd"> environment.</span> |
---|
| 585 | |
---|
| 586 | <span class="sd"> Return</span> |
---|
| 587 | <span class="sd"> ------</span> |
---|
| 588 | |
---|
| 589 | <span class="sd"> '''</span> |
---|
| 590 | <span class="kn">from</span> <span class="nn">genshi.template.text</span> <span class="k">import</span> <span class="n">NewTextTemplate</span> |
---|
| 591 | <span class="kn">from</span> <span class="nn">genshi.template</span> <span class="k">import</span> <span class="n">TemplateLoader</span> |
---|
| 592 | |
---|
| 593 | <span class="n">loader</span> <span class="o">=</span> <span class="n">TemplateLoader</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_TEMPLATES</span><span class="p">,</span> <span class="n">auto_reload</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> |
---|
| 594 | <span class="n">compile_template</span> <span class="o">=</span> <span class="n">loader</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">TEMPFILE_INSTALL_JOB</span><span class="p">,</span> |
---|
| 595 | <span class="bp">cls</span><span class="o">=</span><span class="n">NewTextTemplate</span><span class="p">)</span> |
---|
| 596 | |
---|
| 597 | <span class="n">fp_root_path_to_python</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">fp_root</span><span class="p">,</span> |
---|
| 598 | <span class="n">_config</span><span class="o">.</span><span class="n">FLEXEXTRACT_DIRNAME</span><span class="p">,</span> |
---|
| 599 | <span class="n">_config</span><span class="o">.</span><span class="n">PATH_REL_PYTHON</span><span class="p">)</span> |
---|
| 600 | |
---|
| 601 | <span class="n">stream</span> <span class="o">=</span> <span class="n">compile_template</span><span class="o">.</span><span class="n">generate</span><span class="p">(</span> |
---|
| 602 | <span class="n">usergroup</span> <span class="o">=</span> <span class="n">ecgid</span><span class="p">,</span> |
---|
| 603 | <span class="n">username</span> <span class="o">=</span> <span class="n">ecuid</span><span class="p">,</span> |
---|
| 604 | <span class="n">version_number</span> <span class="o">=</span> <span class="n">_config</span><span class="o">.</span><span class="n">_VERSION_STR</span><span class="p">,</span> |
---|
| 605 | <span class="n">fp_root_path</span> <span class="o">=</span> <span class="n">fp_root_path_to_python</span><span class="p">,</span> |
---|
| 606 | <span class="p">)</span> |
---|
| 607 | |
---|
| 608 | <span class="n">tempjobfile</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_TEMPLATES</span><span class="p">,</span> |
---|
| 609 | <span class="n">_config</span><span class="o">.</span><span class="n">TEMPFILE_JOB</span><span class="p">)</span> |
---|
| 610 | |
---|
| 611 | <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">tempjobfile</span><span class="p">,</span> <span class="s1">'w'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span> |
---|
| 612 | <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">stream</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="s1">'text'</span><span class="p">))</span> |
---|
| 613 | |
---|
| 614 | <span class="k">return</span></div> |
---|
| 615 | |
---|
| 616 | <div class="viewcode-block" id="delete_convert_build"><a class="viewcode-back" href="../api.html#install.delete_convert_build">[docs]</a><span class="k">def</span> <span class="nf">delete_convert_build</span><span class="p">(</span><span class="n">src_path</span><span class="p">):</span> |
---|
| 617 | <span class="sd">'''Clean up the Fortran source directory and remove all</span> |
---|
| 618 | <span class="sd"> build files (e.g. \*.o, \*.mod and CONVERT2)</span> |
---|
| 619 | |
---|
| 620 | <span class="sd"> Parameters</span> |
---|
| 621 | <span class="sd"> ----------</span> |
---|
| 622 | <span class="sd"> src_path : :obj:`string`</span> |
---|
| 623 | <span class="sd"> Path to the fortran source directory.</span> |
---|
| 624 | |
---|
| 625 | <span class="sd"> Return</span> |
---|
| 626 | <span class="sd"> ------</span> |
---|
| 627 | |
---|
| 628 | <span class="sd"> '''</span> |
---|
| 629 | |
---|
| 630 | <span class="n">modfiles</span> <span class="o">=</span> <span class="n">UioFiles</span><span class="p">(</span><span class="n">src_path</span><span class="p">,</span> <span class="s1">'*.mod'</span><span class="p">)</span> |
---|
| 631 | <span class="n">objfiles</span> <span class="o">=</span> <span class="n">UioFiles</span><span class="p">(</span><span class="n">src_path</span><span class="p">,</span> <span class="s1">'*.o'</span><span class="p">)</span> |
---|
| 632 | <span class="n">exefile</span> <span class="o">=</span> <span class="n">UioFiles</span><span class="p">(</span><span class="n">src_path</span><span class="p">,</span> <span class="n">_config</span><span class="o">.</span><span class="n">FORTRAN_EXECUTABLE</span><span class="p">)</span> |
---|
| 633 | |
---|
| 634 | <span class="n">modfiles</span><span class="o">.</span><span class="n">delete_files</span><span class="p">()</span> |
---|
| 635 | <span class="n">objfiles</span><span class="o">.</span><span class="n">delete_files</span><span class="p">()</span> |
---|
| 636 | <span class="n">exefile</span><span class="o">.</span><span class="n">delete_files</span><span class="p">()</span> |
---|
| 637 | |
---|
| 638 | <span class="k">return</span></div> |
---|
| 639 | |
---|
| 640 | <div class="viewcode-block" id="make_convert_build"><a class="viewcode-back" href="../api.html#install.make_convert_build">[docs]</a><span class="k">def</span> <span class="nf">make_convert_build</span><span class="p">(</span><span class="n">src_path</span><span class="p">,</span> <span class="n">makefile</span><span class="p">):</span> |
---|
| 641 | <span class="sd">'''Compiles the Fortran code and generates the executable.</span> |
---|
| 642 | |
---|
| 643 | <span class="sd"> Parameters</span> |
---|
| 644 | <span class="sd"> ----------</span> |
---|
| 645 | <span class="sd"> src_path : :obj:`string`</span> |
---|
| 646 | <span class="sd"> Path to the fortran source directory.</span> |
---|
| 647 | |
---|
| 648 | <span class="sd"> makefile : :obj:`string`</span> |
---|
| 649 | <span class="sd"> The name of the makefile which should be used.</span> |
---|
| 650 | |
---|
| 651 | <span class="sd"> Return</span> |
---|
| 652 | <span class="sd"> ------</span> |
---|
| 653 | |
---|
| 654 | <span class="sd"> '''</span> |
---|
| 655 | |
---|
| 656 | <span class="k">try</span><span class="p">:</span> |
---|
| 657 | <span class="nb">print</span><span class="p">(</span><span class="s1">'Using makefile: '</span> <span class="o">+</span> <span class="n">makefile</span><span class="p">)</span> |
---|
| 658 | <span class="n">p</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">([</span><span class="s1">'make'</span><span class="p">,</span> <span class="s1">'-f'</span><span class="p">,</span> |
---|
| 659 | <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">src_path</span><span class="p">,</span> <span class="n">makefile</span><span class="p">)],</span> |
---|
| 660 | <span class="n">stdin</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span> |
---|
| 661 | <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span> |
---|
| 662 | <span class="n">stderr</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span> |
---|
| 663 | <span class="n">bufsize</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> |
---|
| 664 | <span class="n">pout</span><span class="p">,</span> <span class="n">perr</span> <span class="o">=</span> <span class="n">p</span><span class="o">.</span><span class="n">communicate</span><span class="p">()</span> |
---|
| 665 | <span class="nb">print</span><span class="p">(</span><span class="n">pout</span><span class="p">)</span> |
---|
| 666 | <span class="k">if</span> <span class="n">p</span><span class="o">.</span><span class="n">returncode</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span> |
---|
| 667 | <span class="nb">print</span><span class="p">(</span><span class="n">perr</span><span class="p">)</span> |
---|
| 668 | <span class="nb">print</span><span class="p">(</span><span class="s1">'Please edit '</span> <span class="o">+</span> <span class="n">makefile</span> <span class="o">+</span> |
---|
| 669 | <span class="s1">' or try another Makefile in the src directory.'</span><span class="p">)</span> |
---|
| 670 | <span class="nb">print</span><span class="p">(</span><span class="s1">'Most likely GRIB_API_INCLUDE_DIR, GRIB_API_LIB '</span> |
---|
| 671 | <span class="s1">'and EMOSLIB must be adapted.'</span><span class="p">)</span> |
---|
| 672 | <span class="nb">print</span><span class="p">(</span><span class="s1">'Available Makefiles:'</span><span class="p">)</span> |
---|
| 673 | <span class="nb">print</span><span class="p">(</span><span class="n">UioFiles</span><span class="p">(</span><span class="n">src_path</span><span class="p">,</span> <span class="s1">'Makefile*'</span><span class="p">))</span> |
---|
| 674 | <span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="s1">'Compilation failed!'</span><span class="p">)</span> |
---|
| 675 | <span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> |
---|
| 676 | <span class="nb">print</span><span class="p">(</span><span class="s1">'ERROR: Makefile call failed:'</span><span class="p">)</span> |
---|
| 677 | <span class="nb">print</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> |
---|
| 678 | <span class="k">else</span><span class="p">:</span> |
---|
| 679 | <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">([</span><span class="s1">'ls'</span><span class="p">,</span> <span class="s1">'-l'</span><span class="p">,</span> |
---|
| 680 | <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">src_path</span><span class="p">,</span> |
---|
| 681 | <span class="n">_config</span><span class="o">.</span><span class="n">FORTRAN_EXECUTABLE</span><span class="p">)])</span> |
---|
| 682 | |
---|
| 683 | <span class="k">return</span></div> |
---|
| 684 | |
---|
| 685 | |
---|
| 686 | <span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"__main__"</span><span class="p">:</span> |
---|
| 687 | <span class="n">main</span><span class="p">()</span> |
---|
| 688 | </pre></div> |
---|
| 689 | |
---|
| 690 | </div> |
---|
| 691 | |
---|
| 692 | </div> |
---|
| 693 | <footer> |
---|
| 694 | |
---|
| 695 | |
---|
| 696 | <hr/> |
---|
| 697 | |
---|
| 698 | <div role="contentinfo"> |
---|
| 699 | <p> |
---|
| 700 | © Copyright 2018, Anne Philipp and Leopold Haimberger. |
---|
| 701 | |
---|
| 702 | </p> |
---|
| 703 | </div> |
---|
| 704 | Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. |
---|
| 705 | |
---|
| 706 | </footer> |
---|
| 707 | |
---|
| 708 | </div> |
---|
| 709 | </div> |
---|
| 710 | |
---|
| 711 | </section> |
---|
| 712 | |
---|
| 713 | </div> |
---|
| 714 | |
---|
| 715 | |
---|
| 716 | |
---|
| 717 | |
---|
| 718 | |
---|
| 719 | <script type="text/javascript"> |
---|
| 720 | var DOCUMENTATION_OPTIONS = { |
---|
| 721 | URL_ROOT:'../', |
---|
| 722 | VERSION:'7.1 alpha', |
---|
| 723 | LANGUAGE:'None', |
---|
| 724 | COLLAPSE_INDEX:false, |
---|
| 725 | FILE_SUFFIX:'.html', |
---|
| 726 | HAS_SOURCE: true, |
---|
| 727 | SOURCELINK_SUFFIX: '.txt' |
---|
| 728 | }; |
---|
| 729 | </script> |
---|
| 730 | <script type="text/javascript" src="../_static/jquery.js"></script> |
---|
| 731 | <script type="text/javascript" src="../_static/underscore.js"></script> |
---|
| 732 | <script type="text/javascript" src="../_static/doctools.js"></script> |
---|
| 733 | <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script> |
---|
| 734 | |
---|
| 735 | |
---|
| 736 | |
---|
| 737 | <script type="text/javascript" src="../_static/js/theme.js"></script> |
---|
| 738 | |
---|
| 739 | <script type="text/javascript"> |
---|
| 740 | jQuery(function () { |
---|
| 741 | SphinxRtdTheme.Navigation.enable(true); |
---|
| 742 | }); |
---|
| 743 | </script> |
---|
| 744 | |
---|
| 745 | </body> |
---|
| 746 | </html> |
---|