source: flex_extract.git/Documentation/html/Installation/gateway.html @ 30f7911

ctbtodev
Last change on this file since 30f7911 was 30f7911, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

reviewed installation section of online documentation; minor corrections
  • Property mode set to 100644
File size: 18.3 KB
Line 
1
2
3<!DOCTYPE html>
4<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
5<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
6<head>
7  <meta charset="utf-8">
8 
9  <meta name="viewport" content="width=device-width, initial-scale=1.0">
10 
11  <title>Gateway mode installation &mdash; flex_extract 7.1 alpha documentation</title>
12 
13
14 
15 
16 
17 
18
19 
20  <script type="text/javascript" src="../_static/js/modernizr.min.js"></script>
21 
22   
23      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
24        <script type="text/javascript" src="../_static/jquery.js"></script>
25        <script type="text/javascript" src="../_static/underscore.js"></script>
26        <script type="text/javascript" src="../_static/doctools.js"></script>
27        <script type="text/javascript" src="../_static/language_data.js"></script>
28        <script async="async" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
29   
30    <script type="text/javascript" src="../_static/js/theme.js"></script>
31
32   
33
34 
35  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
36  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
37  <link rel="stylesheet" href="../_static/css/custom.css" type="text/css" />
38  <link rel="stylesheet" href="../_static/css/theme_overrides.css" type="text/css" />
39    <link rel="index" title="Index" href="../genindex.html" />
40    <link rel="search" title="Search" href="../search.html" />
41    <link rel="next" title="Local mode installation" href="local.html" />
42    <link rel="prev" title="Remote mode installation" href="remote.html" /> 
43</head>
44
45<body class="wy-body-for-nav">
46
47   
48  <div class="wy-grid-for-nav">
49   
50    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
51      <div class="wy-side-scroll">
52        <div class="wy-side-nav-search" >
53         
54
55         
56            <a href="../index.html" class="icon icon-home"> flex_extract
57         
58
59         
60          </a>
61
62         
63           
64           
65              <div class="version">
66                7.1
67              </div>
68           
69         
70
71         
72<div role="search">
73  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
74    <input type="text" name="q" placeholder="Search docs" />
75    <input type="hidden" name="check_keywords" value="yes" />
76    <input type="hidden" name="area" value="default" />
77  </form>
78</div>
79
80         
81        </div>
82
83        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
84         
85           
86           
87             
88           
89           
90              <p class="caption"><span class="caption-text">Table of Contents:</span></p>
91<ul class="current">
92<li class="toctree-l1 current"><a class="reference internal" href="../installation.html">Installation</a><ul class="current">
93<li class="toctree-l2"><a class="reference internal" href="../installation.html#registration-at-ecmwf">Registration at ECMWF</a></li>
94<li class="toctree-l2"><a class="reference internal" href="../installation.html#licence-agreement-for-public-datasets">Licence agreement for public datasets</a></li>
95<li class="toctree-l2"><a class="reference internal" href="../installation.html#download-flex-extract">Download <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code></a></li>
96<li class="toctree-l2"><a class="reference internal" href="../installation.html#environment-requirements">Environment requirements</a></li>
97<li class="toctree-l2 current"><a class="reference internal" href="../installation.html#installation-of-flex-extract">Installation of <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code></a><ul class="current">
98<li class="toctree-l3"><a class="reference internal" href="remote.html">Remote mode installation</a></li>
99<li class="toctree-l3 current"><a class="current reference internal" href="#">Gateway mode installation</a><ul>
100<li class="toctree-l4"><a class="reference internal" href="#gateway-environment-requirements">Gateway environment requirements</a></li>
101<li class="toctree-l4"><a class="reference internal" href="#prepare-gateway-environment">Prepare gateway environment</a></li>
102<li class="toctree-l4"><a class="reference internal" href="#test-gateway-environment">Test gateway environment</a></li>
103<li class="toctree-l4"><a class="reference internal" href="#gateway-installation">Gateway installation</a></li>
104</ul>
105</li>
106<li class="toctree-l3"><a class="reference internal" href="local.html">Local mode installation</a></li>
107</ul>
108</li>
109<li class="toctree-l2"><a class="reference internal" href="../installation.html#flex-extract-in-combination-with-flexpart"><code class="docutils literal notranslate"><span class="pre">Flex_extract</span></code> in combination with <code class="docutils literal notranslate"><span class="pre">FLEXPART</span></code></a></li>
110<li class="toctree-l2"><a class="reference internal" href="../installation.html#test-installation">Test installation</a></li>
111</ul>
112</li>
113<li class="toctree-l1"><a class="reference internal" href="../quick_start.html">Quick Start</a></li>
114<li class="toctree-l1"><a class="reference internal" href="../ecmwf_data.html">ECMWF Data</a></li>
115<li class="toctree-l1"><a class="reference internal" href="../documentation.html">Documentation</a></li>
116<li class="toctree-l1"><a class="reference internal" href="../evaluation.html">Evaluation</a></li>
117<li class="toctree-l1"><a class="reference internal" href="../dev_guide.html">Developer Guide</a></li>
118<li class="toctree-l1"><a class="reference internal" href="../changelog.html">Changelog</a></li>
119<li class="toctree-l1"><a class="reference internal" href="../support.html">Support</a></li>
120<li class="toctree-l1"><a class="reference internal" href="../Support/faq.html">FAQ - Frequently asked questions</a></li>
121<li class="toctree-l1"><a class="reference internal" href="../authors.html">Developer Team</a></li>
122</ul>
123
124           
125         
126        </div>
127      </div>
128    </nav>
129
130    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
131
132     
133      <nav class="wy-nav-top" aria-label="top navigation">
134       
135          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
136          <a href="../index.html">flex_extract</a>
137       
138      </nav>
139
140
141      <div class="wy-nav-content">
142       
143        <div class="rst-content">
144       
145         
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161<div role="navigation" aria-label="breadcrumbs navigation">
162
163  <ul class="wy-breadcrumbs">
164   
165      <li><a href="../index.html">Docs</a> &raquo;</li>
166       
167          <li><a href="../installation.html">Installation</a> &raquo;</li>
168       
169      <li>Gateway mode installation</li>
170   
171   
172      <li class="wy-breadcrumbs-aside">
173       
174           
175            <a href="../_sources/Installation/gateway.rst.txt" rel="nofollow"> View page source</a>
176         
177       
178      </li>
179   
180  </ul>
181
182 
183  <hr/>
184</div>
185          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
186           <div itemprop="articleBody">
187           
188  <div class="section" id="gateway-mode-installation">
189<h1>Gateway mode installation<a class="headerlink" href="#gateway-mode-installation" title="Permalink to this headline"></a></h1>
190<div class="toctree-wrapper compound">
191</div>
192<div class="section" id="gateway-environment-requirements">
193<span id="ref-req-gateway"></span><span id="ref-gateway-mode"></span><h2>Gateway environment requirements<a class="headerlink" href="#gateway-environment-requirements" title="Permalink to this headline"></a></h2>
194<p>The environment on your local system has to provide these software packages
195and libraries, since <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code> does only prepare the job script and send
196it to the ECMWF servers:</p>
197<ul class="simple">
198<li><a class="reference external" href="https://www.python.org/downloads/">Python3</a> or <a class="reference external" href="https://www.anaconda.com/distribution/#download-section">Anaconda Python3</a></li>
199<li><a class="reference external" href="http://www.numpy.org/">numpy</a></li>
200<li><a class="reference external" href="https://genshi.edgewall.org/">genshi</a></li>
201</ul>
202</div>
203<div class="section" id="prepare-gateway-environment">
204<span id="ref-prep-gateway"></span><h2>Prepare gateway environment<a class="headerlink" href="#prepare-gateway-environment" title="Permalink to this headline"></a></h2>
205<p>The easiest way to install all required packages is to use the package management system of your Linux distribution. To do so, it is necessary to use a user with admin rights.
206The installation was tested on a <em>GNU/Linux Debian buster</em> and an <em>Ubuntu 18.04 Bionic Beaver</em> system.</p>
207<div class="highlight-sh notranslate"><div class="highlight"><pre><span></span><span class="c1"># On a Linux Debian or Ubuntu system do</span>
208<span class="c1"># (if not already available):</span>
209apt-get install python3
210apt-get install pip
211apt-get install genshi
212apt-get install numpy
213</pre></div>
214</div>
215</div>
216<div class="section" id="test-gateway-environment">
217<span id="ref-test-gateway"></span><h2>Test gateway environment<a class="headerlink" href="#test-gateway-environment" title="Permalink to this headline"></a></h2>
218<p>Check the availability of the python packages by typing <code class="docutils literal notranslate"><span class="pre">python3</span></code> in
219a terminal window and run the <code class="docutils literal notranslate"><span class="pre">import</span></code> commands in the python shell.
220If there are no error messages, you succeeded in setting up the environment.</p>
221<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="c1"># check in python3 console</span>
222<span class="kn">import</span> <span class="nn">genshi</span>
223<span class="kn">import</span> <span class="nn">numpy</span>
224</pre></div>
225</div>
226</div>
227<div class="section" id="gateway-installation">
228<span id="ref-install-gateway"></span><h2>Gateway installation<a class="headerlink" href="#gateway-installation" title="Permalink to this headline"></a></h2>
229<p>In this mode, access to the ECMWF computing and archiving facilities is enabled through an ECaccess gateway server on a local member state server. The ECaccess framework is necessary to interactively submit jobs to the ECMWF batch system and to transfer files between ECMWF and local gateway server.</p>
230<p>Please see <a class="reference external" href="https://confluence.ecmwf.int/display/ECAC/ECaccess+Home">ECMWF’s instructions on gateway server</a> to establish the gateway server if not already in place.
231Additionally, to be able to use the Ecaccess file transfer service <strong>ectrans</strong> please also create an association.
232The easiest way is to visit the ECaccess Member State Gateway website (e.g. msgatway.ecmwf.int) and follow the instructions in the short <a class="reference external" href="https://confluence.ecmwf.int/download/attachments/45759146/ECaccess.pdf">ECaccess Presentation</a> (page 17 ff.). Additional documentation can be found on the <a class="reference external" href="https://confluence.ecmwf.int/display/ECAC/Unattended+file+transfer+-+ectrans">ECMWF ectrans site</a>.</p>
233<p>After everything is set up you have to create an <em>ecaccess certificate</em> to be able to send and receive files from and to the ECMWF server. You can do this by using the <code class="docutils literal notranslate"><span class="pre">ecaccess-certificate-create</span></code> command on the gateway server. You will be prompted for your ECMWF member state user name and a password (which will be usually generated by a Token). This certificate has to be re-newed periodically (every 7 days).</p>
234<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>$ ecaccess-certificate-create
235Please enter your user-id: example_username
236Your passcode: ***
237</pre></div>
238</div>
239<p><code class="docutils literal notranslate"><span class="pre">Flex_extract</span></code> will be run on an ECMWF server which makes the setup the same as for the <strong>remote mode</strong>. In the <code class="docutils literal notranslate"><span class="pre">setup.sh</span></code> script <a class="reference external" href="Documentation/Input/setup.html">[ref]</a>, select the <code class="docutils literal notranslate"><span class="pre">Makefile.gfortran</span></code> for the <code class="docutils literal notranslate"><span class="pre">CONVERT2</span></code> Fortran program and the ECMWF server (<em>target</em>) you would like to use.
240The job script, send to the job queue via the ECaccess software, selects again automatically the correct libraries from the module system. For enableing the file transfer you have to set the <em>ECUID</em>, <em>ECGID</em>, <em>GATEWAY</em> and <em>DESTINATION</em> parameter values.</p>
241<div class="literal-block-wrapper docutils container" id="setup-sh">
242<div class="code-block-caption"><span class="caption-text">‘Example settings for a gateway installation.’</span><a class="headerlink" href="#setup-sh" title="Permalink to this code"></a></div>
243<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># -----------------------------------------------------------------</span>
244<span class="c1"># AVAILABLE COMMANDLINE ARGUMENTS TO SET</span>
245<span class="c1">#</span>
246<span class="c1"># THE USER HAS TO SPECIFY THESE PARAMETER</span>
247<span class="c1">#</span>
248<span class="nv">TARGET</span><span class="o">=</span><span class="s1">&#39;ecgate&#39;</span>
249<span class="nv">MAKEFILE</span><span class="o">=</span><span class="s1">&#39;Makefile.gfortran&#39;</span>
250<span class="nv">ECUID</span><span class="o">=</span><span class="s1">&#39;uid&#39;</span>
251<span class="nv">ECGID</span><span class="o">=</span><span class="s1">&#39;gid&#39;</span>
252<span class="nv">GATEWAY</span><span class="o">=</span><span class="s1">&#39;server.example.edu&#39;</span>
253<span class="nv">DESTINATION</span><span class="o">=</span><span class="s1">&#39;example@genericSftp&#39;</span>
254<span class="nv">INSTALLDIR</span><span class="o">=</span>None
255<span class="nv">JOB_TEMPLATE</span><span class="o">=</span><span class="s1">&#39;job.template&#39;</span>
256<span class="nv">CONTROLFILE</span><span class="o">=</span><span class="s1">&#39;CONTROL_EA5&#39;</span>
257</pre></div>
258</div>
259</div>
260<p>Afterwards, type:</p>
261<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>$ ./setup.sh
262</pre></div>
263</div>
264<p>to start the installation. You should see the following output at the command line.</p>
265<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Output of setup.sh</span>
266Create tarball ...
267Job compilation script has been submitted to ecgate <span class="k">for</span> installation in <span class="si">${</span><span class="nv">HOME</span><span class="si">}</span>/flex_extract_vX.X
268You should get an email with subject <span class="s2">&quot;flexcompile&quot;</span> within the next few minutes!
269</pre></div>
270</div>
271<p>The email content should look like this with a “SUCCESS” statement in the last line:</p>
272<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>gfortran   -m64 -fdefault-real-8 -fcray-pointer -fno-second-underscore  -ffixed-line-length-132 -fopenmp  -fconvert<span class="o">=</span>big-endian  -c -g -O3 -fopenmp phgrreal.f
273gfortran   -m64 -fdefault-real-8 -fcray-pointer -fno-second-underscore  -ffixed-line-length-132 -fopenmp  -fconvert<span class="o">=</span>big-endian  -c -g -O3 -fopenmp grphreal.f
274gfortran   -m64 -fdefault-real-8 -fcray-pointer -fno-second-underscore  -ffixed-line-length-132 -fopenmp  -fconvert<span class="o">=</span>big-endian  -c -g -O3 -fopenmp ftrafo.f
275gfortran   -m64 -fdefault-real-8 -fcray-pointer -fno-second-underscore  -ffixed-line-length-132 -fopenmp  -fconvert<span class="o">=</span>big-endian  -c -O3 -I. -I/usr/local/apps/eccodes/2.12.0/GNU/6.3.0/include -g rwGRIB2.f90
276gfortran   -m64 -fdefault-real-8 -fcray-pointer -fno-second-underscore  -ffixed-line-length-132 -fopenmp  -fconvert<span class="o">=</span>big-endian  -c -O3 -I. -I/usr/local/apps/eccodes/2.12.0/GNU/6.3.0/include -g posnam.f
277gfortran   -m64 -fdefault-real-8 -fcray-pointer -fno-second-underscore  -ffixed-line-length-132 -fopenmp  -fconvert<span class="o">=</span>big-endian  -c -O3 -I. -I/usr/local/apps/eccodes/2.12.0/GNU/6.3.0/include -g preconvert.f90
278gfortran   -m64 -fdefault-real-8 -fcray-pointer -fno-second-underscore  -ffixed-line-length-132 -fopenmp  -fconvert<span class="o">=</span>big-endian  -g -O3 -o ./CONVERT2 ftrafo.o phgrreal.o grphreal.o rwGRIB2.o posnam.o preconvert.o -L/usr/local/apps/eccodes/2.12.0/GNU/6.3.0/lib -Wl,-rpath,/usr/local/apps/eccodes/2.12.0/GNU/6.3.0/lib -leccodes_f90 -leccodes -ljasper -lpthread -L/usr/local/apps/jasper/1.900.1/LP64/lib -ljasper -lm -L/usr/local/apps/libemos/000455/GNU/6.3.0/lib  -Wl,-rpath,/usr/local/apps/libemos/000455/GNU/6.3.0/lib  -lemos.R64.D64.I32 -L/usr/local/apps/fftw/3.3.4/GNU/6.3.0/lib -Wl,-rpath,/usr/local/apps/fftw/3.3.4/GNU/6.3.0/lib -lfftw3
279-rwxr-x---. <span class="m">1</span> USER at <span class="m">353134</span> May <span class="m">23</span> <span class="m">12</span>:27 CONVERT2
280SUCCESS!
281</pre></div>
282</div>
283</div>
284</div>
285
286
287           </div>
288           
289          </div>
290          <footer>
291 
292    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
293     
294        <a href="local.html" class="btn btn-neutral float-right" title="Local mode installation" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
295     
296     
297        <a href="remote.html" class="btn btn-neutral float-left" title="Remote mode installation" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
298     
299    </div>
300 
301
302  <hr/>
303
304  <div role="contentinfo">
305    <p>
306        &copy; Copyright 2019, Anne Philipp and Leopold Haimberger
307
308    </p>
309  </div>
310  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
311
312</footer>
313
314        </div>
315      </div>
316
317    </section>
318
319  </div>
320 
321
322
323  <script type="text/javascript">
324      jQuery(function () {
325          SphinxRtdTheme.Navigation.enable(true);
326      });
327  </script>
328
329 
330 
331   
332   
333
334</body>
335</html>
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG