source: flex_extract.git/Documentation/html/quick_start.html @ 47be0cb6

ctbtodev
Last change on this file since 47be0cb6 was 30f7911, checked in by Anne Philipp <anne.philipp@…>, 5 years ago

reviewed installation section of online documentation; minor corrections
  • Property mode set to 100644
File size: 59.2 KB
Line 
1
2
3<!DOCTYPE html>
4<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
5<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
6<head>
7  <meta charset="utf-8">
8 
9  <meta name="viewport" content="width=device-width, initial-scale=1.0">
10 
11  <title>Quick Start &mdash; flex_extract 7.1 alpha documentation</title>
12 
13
14 
15 
16 
17 
18
19 
20  <script type="text/javascript" src="_static/js/modernizr.min.js"></script>
21 
22   
23      <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
24        <script type="text/javascript" src="_static/jquery.js"></script>
25        <script type="text/javascript" src="_static/underscore.js"></script>
26        <script type="text/javascript" src="_static/doctools.js"></script>
27        <script type="text/javascript" src="_static/language_data.js"></script>
28        <script async="async" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
29   
30    <script type="text/javascript" src="_static/js/theme.js"></script>
31
32   
33
34 
35  <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
36  <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
37  <link rel="stylesheet" href="_static/css/custom.css" type="text/css" />
38  <link rel="stylesheet" href="_static/css/theme_overrides.css" type="text/css" />
39    <link rel="index" title="Index" href="genindex.html" />
40    <link rel="search" title="Search" href="search.html" />
41    <link rel="next" title="ECMWF Data" href="ecmwf_data.html" />
42    <link rel="prev" title="Local mode installation" href="Installation/local.html" /> 
43</head>
44
45<body class="wy-body-for-nav">
46
47   
48  <div class="wy-grid-for-nav">
49   
50    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
51      <div class="wy-side-scroll">
52        <div class="wy-side-nav-search" >
53         
54
55         
56            <a href="index.html" class="icon icon-home"> flex_extract
57         
58
59         
60          </a>
61
62         
63           
64           
65              <div class="version">
66                7.1
67              </div>
68           
69         
70
71         
72<div role="search">
73  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
74    <input type="text" name="q" placeholder="Search docs" />
75    <input type="hidden" name="check_keywords" value="yes" />
76    <input type="hidden" name="area" value="default" />
77  </form>
78</div>
79
80         
81        </div>
82
83        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
84         
85           
86           
87             
88           
89           
90              <p class="caption"><span class="caption-text">Table of Contents:</span></p>
91<ul class="current">
92<li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li>
93<li class="toctree-l1 current"><a class="current reference internal" href="#">Quick Start</a><ul>
94<li class="toctree-l2"><a class="reference internal" href="#job-preparation">Job preparation</a><ul>
95<li class="toctree-l3"><a class="reference internal" href="#remote-and-gateway-modes">Remote and gateway modes</a></li>
96<li class="toctree-l3"><a class="reference internal" href="#local-mode">Local mode</a></li>
97</ul>
98</li>
99<li class="toctree-l2"><a class="reference internal" href="#selection-and-adjustment-of-control-files">Selection and adjustment of <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> files</a><ul>
100<li class="toctree-l3"><a class="reference internal" href="#public-datasets">Public datasets</a></li>
101<li class="toctree-l3"><a class="reference internal" href="#cera">CERA</a></li>
102<li class="toctree-l3"><a class="reference internal" href="#era-5">ERA 5</a></li>
103<li class="toctree-l3"><a class="reference internal" href="#era-interim">ERA-Interim</a></li>
104<li class="toctree-l3"><a class="reference internal" href="#operational-data">Operational data</a></li>
105<li class="toctree-l3"><a class="reference internal" href="#specific-features">Specific features</a></li>
106<li class="toctree-l3"><a class="reference internal" href="#hints-for-definition-of-some-parameter-combinations">Hints for definition of some parameter combinations</a><ul class="simple">
107</ul>
108</li>
109</ul>
110</li>
111</ul>
112</li>
113<li class="toctree-l1"><a class="reference internal" href="ecmwf_data.html">ECMWF Data</a></li>
114<li class="toctree-l1"><a class="reference internal" href="documentation.html">Documentation</a></li>
115<li class="toctree-l1"><a class="reference internal" href="evaluation.html">Evaluation</a></li>
116<li class="toctree-l1"><a class="reference internal" href="dev_guide.html">Developer Guide</a></li>
117<li class="toctree-l1"><a class="reference internal" href="changelog.html">Changelog</a></li>
118<li class="toctree-l1"><a class="reference internal" href="support.html">Support</a></li>
119<li class="toctree-l1"><a class="reference internal" href="Support/faq.html">FAQ - Frequently asked questions</a></li>
120<li class="toctree-l1"><a class="reference internal" href="authors.html">Developer Team</a></li>
121</ul>
122
123           
124         
125        </div>
126      </div>
127    </nav>
128
129    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
130
131     
132      <nav class="wy-nav-top" aria-label="top navigation">
133       
134          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
135          <a href="index.html">flex_extract</a>
136       
137      </nav>
138
139
140      <div class="wy-nav-content">
141       
142        <div class="rst-content">
143       
144         
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160<div role="navigation" aria-label="breadcrumbs navigation">
161
162  <ul class="wy-breadcrumbs">
163   
164      <li><a href="index.html">Docs</a> &raquo;</li>
165       
166      <li>Quick Start</li>
167   
168   
169      <li class="wy-breadcrumbs-aside">
170       
171           
172            <a href="_sources/quick_start.rst.txt" rel="nofollow"> View page source</a>
173         
174       
175      </li>
176   
177  </ul>
178
179 
180  <hr/>
181</div>
182          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
183           <div itemprop="articleBody">
184           
185  <div class="section" id="quick-start">
186<h1>Quick Start<a class="headerlink" href="#quick-start" title="Permalink to this headline"></a></h1>
187<p><code class="docutils literal notranslate"><span class="pre">Flex_extract</span></code> by itself is a command-line tool. With version 7.1 an upstream shell script was implemented which calls <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code> with the command-line parameters.</p>
188<p>To submit a job with <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code> change the working directory to the <code class="docutils literal notranslate"><span class="pre">Run</span></code> directory  which is placed directly under the <code class="docutils literal notranslate"><span class="pre">flex_extract_vX.X</span></code> root directory  (<code class="docutils literal notranslate"><span class="pre">X.X</span></code> is the version number):</p>
189<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">cd</span> &lt;path-to-flex_extract_vX.X&gt;/Run
190</pre></div>
191</div>
192<p>Within this directory you can find everything you need to modify and run <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code>. The following tree shows a shortened list of directories and important files. The <code class="docutils literal notranslate"><span class="pre">*</span></code> serves as a wildcard. The brackets <code class="docutils literal notranslate"><span class="pre">[]</span></code> means that the file appearance depends on the mode of application and night not be present.</p>
193<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>Run
194├── Control
195│   ├── CONTROL_*
196├── Jobscripts
197│   ├── compilejob.ksh
198│   ├── job.ksh
199│   ├── <span class="o">[</span>joboper.ksh<span class="o">]</span>
200├── Workspace
201│   ├── CERA_example
202│   │   ├── CE000908*
203├── <span class="o">[</span>ECMWF_ENV<span class="o">]</span>
204├── run_local.sh
205└── run.sh
206</pre></div>
207</div>
208<p>The <code class="docutils literal notranslate"><span class="pre">Jobscripts</span></code> directory is used to store the Korn shell job scripts generated by a <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code> run in the <strong>Remote</strong> or <strong>Gateway</strong> mode. They are used to submit the setup information to the ECMWF server and start the jobs in ECMWF’s batch mode. The typical user must not touch these files. They will be generated from template files which are stored in the <code class="docutils literal notranslate"><span class="pre">Templates</span></code> directory under <code class="docutils literal notranslate"><span class="pre">flex_extract_vX.X</span></code>. Usually there will be a <code class="docutils literal notranslate"><span class="pre">compilejob.ksh</span></code> and a <code class="docutils literal notranslate"><span class="pre">job.ksh</span></code> script which are explained in the section <a class="reference internal" href="Documentation/input.html"><span class="doc">Control &amp; Input Data</span></a>. In the rare case of operational data extraction there will be a <code class="docutils literal notranslate"><span class="pre">joboper.ksh</span></code> which is designed to get the time parameters from environment variables at the ECMWF servers.</p>
209<p>The <code class="docutils literal notranslate"><span class="pre">Controls</span></code> directory contains a number of example <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> files. These``CONTROL`` files represent the current range of possible dataset retrievals with <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code>. Some parameters in the <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> files can be adapted and some others should not be changed. In this <a class="reference internal" href="#"><span class="doc">Quick Start</span></a> guide we explain how an extraction with <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code> can be started in the different <a class="reference internal" href="Documentation/Overview/app_modes.html"><span class="doc">Application Modes</span></a> and point out some specifics of each dataset and <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> file.</p>
210<p>Directly under <code class="docutils literal notranslate"><span class="pre">Run</span></code> you find the files <code class="docutils literal notranslate"><span class="pre">run.sh</span></code> and <code class="docutils literal notranslate"><span class="pre">run_local.sh</span></code> and according to your selected <a class="reference internal" href="Documentation/Overview/app_modes.html"><span class="doc">Application Modes</span></a> there might also be a file named <code class="docutils literal notranslate"><span class="pre">ECMWF_ENV</span></code> for the user credentials to quickly and automatically access ECMWF servers.</p>
211<p>From version 7.1 on, the <code class="docutils literal notranslate"><span class="pre">run.sh</span></code> (or <code class="docutils literal notranslate"><span class="pre">run_local.sh</span></code>) script is the main access point to <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code>.</p>
212<div class="admonition note">
213<p class="first admonition-title">Note</p>
214<p class="last">Note that, for experienced users (or users of older versions), it is still possible to access <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code> directly via the <code class="docutils literal notranslate"><span class="pre">submit.py</span></code> script which can be found in the directory <code class="docutils literal notranslate"><span class="pre">flex_extract_vX.X/Source/Python</span></code>.</p>
215</div>
216<div class="section" id="job-preparation">
217<h2>Job preparation<a class="headerlink" href="#job-preparation" title="Permalink to this headline"></a></h2>
218<p>To actually start a job with <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code> it is only necessary to submit the <code class="docutils literal notranslate"><span class="pre">run.sh</span></code> or <code class="docutils literal notranslate"><span class="pre">run_local.sh</span></code> script on the command-line. For the specification of dataset and the selection of the access mode it is necessary to select and modify a <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> file and change the parameters in the user section of the <code class="docutils literal notranslate"><span class="pre">run</span></code> scripts. The following sections describes the differences in the application modes and where the results will be stored.</p>
219<div class="section" id="remote-and-gateway-modes">
220<h3>Remote and gateway modes<a class="headerlink" href="#remote-and-gateway-modes" title="Permalink to this headline"></a></h3>
221<p>For member state users it is recommended to use the <em>remote</em> or <em>gateway</em> mode, especially for more demanding tasks, to retrieve and convert data on ECMWF machines and to transfer only the final output files to the local host.</p>
222<dl class="docutils">
223<dt>Remote mode</dt>
224<dd><p class="first">The only difference between both modes is the users working location. In the <em>remote</em> mode you have to login to the ECMWF server and then go to the <code class="docutils literal notranslate"><span class="pre">Run</span></code> directory as shown above. At ECMWF servers <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code> is installed in the <code class="docutils literal notranslate"><span class="pre">$HOME</span></code> directory. However, to be able to start the program you have to load the <code class="docutils literal notranslate"><span class="pre">Python3</span></code> environment with the module system first.</p>
225<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Remote mode</span>
226ssh -X &lt;ecuid&gt;@ecaccess.ecmwf.int
227</pre></div>
228</div>
229<div class="last highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># On ECMWF server</span>
230<span class="o">[</span>&lt;ecuid&gt;@ecgb11 ~<span class="o">]</span>$ module load python3
231<span class="o">[</span>&lt;ecuid&gt;@ecgb11 ~<span class="o">]</span>$ <span class="nb">cd</span> flex_extract_vX.X/Run
232</pre></div>
233</div>
234</dd>
235<dt>Gateway mode</dt>
236<dd><p class="first">For the gateway mode you have to log in on the gateway server and go to the <code class="docutils literal notranslate"><span class="pre">Run</span></code> directory of <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code>:</p>
237<div class="last highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Gateway mode</span>
238ssh &lt;user&gt;@&lt;gatewayserver&gt;
239<span class="nb">cd</span> &lt;path-to-flex_extract_vX.X&gt;/Run
240</pre></div>
241</div>
242</dd>
243</dl>
244<p>From here on the working process is the same for both modes.</p>
245<p>For your first submission you should use one of the example <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> files stored in the <code class="docutils literal notranslate"><span class="pre">Control</span></code> directory. We recommend to extract <em>CERA-20C</em> data since they usually guarantee quick results and are best for testing reasons.</p>
246<p>Therefore open the <code class="docutils literal notranslate"><span class="pre">run.sh</span></code> file and modify the parameter block marked in the file as shown below:</p>
247<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># -----------------------------------------------------------------</span>
248<span class="c1"># AVAILABLE COMMANDLINE ARGUMENTS TO SET</span>
249<span class="c1">#</span>
250<span class="c1"># THE USER HAS TO SPECIFY THESE PARAMETERS:</span>
251
252<span class="nv">QUEUE</span><span class="o">=</span><span class="s1">&#39;ecgate&#39;</span>
253<span class="nv">START_DATE</span><span class="o">=</span>None
254<span class="nv">END_DATE</span><span class="o">=</span>None
255<span class="nv">DATE_CHUNK</span><span class="o">=</span>None
256<span class="nv">JOB_CHUNK</span><span class="o">=</span><span class="m">3</span>
257<span class="nv">BASETIME</span><span class="o">=</span>None
258<span class="nv">STEP</span><span class="o">=</span>None
259<span class="nv">LEVELIST</span><span class="o">=</span>None
260<span class="nv">AREA</span><span class="o">=</span>None
261<span class="nv">INPUTDIR</span><span class="o">=</span>None
262<span class="nv">OUTPUTDIR</span><span class="o">=</span>None
263<span class="nv">PP_ID</span><span class="o">=</span>None
264<span class="nv">JOB_TEMPLATE</span><span class="o">=</span><span class="s1">&#39;job.temp&#39;</span>
265<span class="nv">CONTROLFILE</span><span class="o">=</span><span class="s1">&#39;CONTROL_CERA&#39;</span>
266<span class="nv">DEBUG</span><span class="o">=</span><span class="m">0</span>
267<span class="nv">REQUEST</span><span class="o">=</span><span class="m">2</span>
268<span class="nv">PUBLIC</span><span class="o">=</span><span class="m">0</span>
269</pre></div>
270</div>
271<p>This would retrieve a one day (08.09.2000) <em>CERA-20C</em> dataset with 3 hourly temporal resolution and a small 1° domain over Europe. Since the <code class="docutils literal notranslate"><span class="pre">ectrans</span></code> parameter is set to <code class="docutils literal notranslate"><span class="pre">1</span></code> the resulting output files will be transferred to the local gateway into the path stored in the destination (SEE INSTRUCTIONS FROM INSTALLATION).  The parameters listed in the <code class="docutils literal notranslate"><span class="pre">run.sh</span></code> file would overwrite existing settings in the <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> file.</p>
272<p>To start the retrieval you only have to start the script by:</p>
273<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>./run.sh
274</pre></div>
275</div>
276<p><code class="docutils literal notranslate"><span class="pre">Flex_extract</span></code> will print some information about the job. If there is no error in the submission to the ECMWF server you will see something like this:</p>
277<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>---- On-demand mode! ----
278The job id is: <span class="m">10627807</span>
279You should get an email per job with subject flex.hostname.pid
280FLEX_EXTRACT JOB SCRIPT IS SUBMITED!
281</pre></div>
282</div>
283<p>Once submitted you can check the progress of the submitted job using <code class="docutils literal notranslate"><span class="pre">ecaccess-job-list</span></code>. You should get an email after the job is finished with a detailed protocol of what was done.</p>
284<p>In case the job fails you will receive an email with the subject <code class="docutils literal notranslate"><span class="pre">ERROR!</span></code> and the job name. You can then check for information in the email or you can check on ECMWF server in the <code class="docutils literal notranslate"><span class="pre">$SCRATCH</span></code> directory for debugging information.</p>
285<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nb">cd</span> <span class="nv">$SCRATCH</span>
286ls -rthl
287</pre></div>
288</div>
289<p>The last command lists the most recent logs and temporary retrieval directories (usually <code class="docutils literal notranslate"><span class="pre">pythonXXXXX</span></code>, where XXXXX is the process id). Under <code class="docutils literal notranslate"><span class="pre">pythonXXXXX</span></code> a copy of the <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> file is stored under the name <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code>, the protocol is stored in the file <code class="docutils literal notranslate"><span class="pre">prot</span></code> and the temporary files as well as the resulting files are stored in a directory <code class="docutils literal notranslate"><span class="pre">work</span></code>. The original name of the <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> file is stored in this new file under parameter <code class="docutils literal notranslate"><span class="pre">controlfile</span></code>.</p>
290<div class="literal-block-wrapper docutils container" id="id3">
291<div class="code-block-caption"><span class="caption-text">“Example structure of <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code> output directory on ECMWF servers.”</span><a class="headerlink" href="#id3" title="Permalink to this code"></a></div>
292<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>pythonXXXXX
293├── CONTROL
294├── prot
295├── work
296│   ├── temporary files
297│   ├── CE000908* <span class="o">(</span>resulting files<span class="o">)</span>
298</pre></div>
299</div>
300</div>
301<p>If the job was submitted to the HPC ( <code class="docutils literal notranslate"><span class="pre">queue=cca</span></code> ) you may login to the HPC and look into the directory <code class="docutils literal notranslate"><span class="pre">/scratch/ms/ECGID/ECUID/.ecaccess_do_not_remove</span></code> for job logs. The working directories are deleted after job failure and thus normally cannot be accessed.</p>
302<p>To check if the resulting files are still transferred to local gateway server you can use the command <code class="docutils literal notranslate"><span class="pre">ecaccess-ectrans-list</span></code> or check the destination path for resulting files on your local gateway server.</p>
303</div>
304<div class="section" id="local-mode">
305<h3>Local mode<a class="headerlink" href="#local-mode" title="Permalink to this headline"></a></h3>
306<p>To get to know the working process and to start your first submission you could use one of the example <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> files stored in the <code class="docutils literal notranslate"><span class="pre">Control</span></code> directory as they are. For quick results and for testing reasons it is recommended to extract <em>CERA-20C</em> data.</p>
307<p>Open the <code class="docutils literal notranslate"><span class="pre">run_local.sh</span></code> file and modify the parameter block marked in the file as shown below. The differences are highlighted.</p>
308<table border="1" class="docutils">
309<colgroup>
310<col width="50%" />
311<col width="50%" />
312</colgroup>
313<tbody valign="top">
314<tr class="row-odd"><td>Take this for <strong>member-state user</strong></td>
315<td>Take this for <strong>public user</strong></td>
316</tr>
317<tr class="row-even"><td><div class="first last highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># -----------------------------------------</span>
318<span class="c1"># AVAILABLE COMMANDLINE ARGUMENTS TO SET</span>
319<span class="c1">#</span>
320<span class="c1"># THE USER HAS TO SPECIFY THESE PARAMETERs:</span>
321<span class="c1">#</span>
322
323<span class="nv">QUEUE</span><span class="o">=</span><span class="s1">&#39;&#39;</span>
324<span class="nv">START_DATE</span><span class="o">=</span>None
325<span class="nv">END_DATE</span><span class="o">=</span>None
326<span class="nv">DATE_CHUNK</span><span class="o">=</span>None
327<span class="nv">JOB_CHUNK</span><span class="o">=</span>None
328<span class="nv">BASETIME</span><span class="o">=</span>None
329<span class="nv">STEP</span><span class="o">=</span>None
330<span class="nv">LEVELIST</span><span class="o">=</span>None
331<span class="nv">AREA</span><span class="o">=</span>None
332<span class="hll"><span class="nv">INPUTDIR</span><span class="o">=</span><span class="s1">&#39;./Workspace/CERA&#39;</span>
333</span><span class="nv">OUTPUTDIR</span><span class="o">=</span>None
334<span class="nv">PP_ID</span><span class="o">=</span>None
335<span class="nv">JOB_TEMPLATE</span><span class="o">=</span><span class="s1">&#39;&#39;</span>
336<span class="hll"><span class="nv">CONTROLFILE</span><span class="o">=</span><span class="s1">&#39;CONTROL_CERA&#39;</span>
337</span><span class="nv">DEBUG</span><span class="o">=</span><span class="m">0</span>
338<span class="nv">REQUEST</span><span class="o">=</span><span class="m">0</span>
339<span class="hll"><span class="nv">PUBLIC</span><span class="o">=</span><span class="m">0</span>
340</span></pre></div>
341</div>
342</td>
343<td><div class="first last highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># -----------------------------------------</span>
344<span class="c1"># AVAILABLE COMMANDLINE ARGUMENTS TO SET</span>
345<span class="c1">#</span>
346<span class="c1"># THE USER HAS TO SPECIFY THESE PARAMETERs:</span>
347<span class="c1">#</span>
348
349<span class="nv">QUEUE</span><span class="o">=</span><span class="s1">&#39;&#39;</span>
350<span class="nv">START_DATE</span><span class="o">=</span>None
351<span class="nv">END_DATE</span><span class="o">=</span>None
352<span class="nv">DATE_CHUNK</span><span class="o">=</span>None
353<span class="nv">JOB_CHUNK</span><span class="o">=</span>None
354<span class="nv">BASETIME</span><span class="o">=</span>None
355<span class="nv">STEP</span><span class="o">=</span>None
356<span class="nv">LEVELIST</span><span class="o">=</span>None
357<span class="nv">AREA</span><span class="o">=</span>None
358<span class="hll"><span class="nv">INPUTDIR</span><span class="o">=</span><span class="s1">&#39;./Workspace/CERApublic&#39;</span>
359</span><span class="nv">OUTPUTDIR</span><span class="o">=</span>None
360<span class="nv">PP_ID</span><span class="o">=</span>None
361<span class="nv">JOB_TEMPLATE</span><span class="o">=</span><span class="s1">&#39;&#39;</span>
362<span class="hll"><span class="nv">CONTROLFILE</span><span class="o">=</span><span class="s1">&#39;CONTROL_CERA.public&#39;</span>
363</span><span class="nv">DEBUG</span><span class="o">=</span><span class="m">0</span>
364<span class="nv">REQUEST</span><span class="o">=</span><span class="m">0</span>
365<span class="hll"><span class="nv">PUBLIC</span><span class="o">=</span><span class="m">1</span>
366</span></pre></div>
367</div>
368</td>
369</tr>
370</tbody>
371</table>
372<p>This would retrieve a one day (08.09.2000) <em>CERA-20C</em> dataset with 3 hourly temporal resolution and a small 1° domain over Europe. The destination location for this retrieval will be within the <code class="docutils literal notranslate"><span class="pre">Workspace</span></code> directory within <code class="docutils literal notranslate"><span class="pre">Run</span></code>. This can be changed to whatever path you like. The parameters listed in <code class="docutils literal notranslate"><span class="pre">run_local.sh</span></code> would overwrite existing settings in the <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> file.</p>
373<p>To start the retrieval you then start the script by:</p>
374<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>./run_local.sh
375</pre></div>
376</div>
377<p>While job submission on the local host is convenient and easy to monitor (on standard output), there are a few caveats with this option:</p>
378<blockquote>
379<div><ol class="arabic simple">
380<li>There is a maximum size of 20GB for single retrieval via ECMWF Web API. Normally this is not a problem but for global fields with T1279 resolution and hourly time steps the limit may already apply.</li>
381<li>If the retrieved MARS files are large but the resulting files are relative small (small local domain) then the retrieval to the local host may be inefficient since all data must be transferred via the Internet. This scenario applies most notably if <code class="docutils literal notranslate"><span class="pre">etadot</span></code> has to be calculated via the continuity equation as this requires global fields even if the domain is local. In this case job submission via ecgate might be a better choice. It really depends on the use patterns and also on the internet connection speed.</li>
382</ol>
383</div></blockquote>
384</div>
385</div>
386<div class="section" id="selection-and-adjustment-of-control-files">
387<h2>Selection and adjustment of <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> files<a class="headerlink" href="#selection-and-adjustment-of-control-files" title="Permalink to this headline"></a></h2>
388<p>This section describes how to work with the <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> files. A detailed explanation of <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> file parameters and naming compositions can be found <a class="reference external" href="Documentation/Input/control.html">here</a>. The more accurately the <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> file describes the retrieval needed, the fewer command-line parameters are needed to be set in the <code class="docutils literal notranslate"><span class="pre">run</span></code> scripts. With version <code class="docutils literal notranslate"><span class="pre">7.1</span></code> all <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> file parameters have default values. They can be found in section <a class="reference external" href="Documentation/Input/control_params.html">CONTROL parameters</a> or in the <code class="docutils literal notranslate"><span class="pre">CONTROL.documentation</span></code> file within the <code class="docutils literal notranslate"><span class="pre">Control</span></code> directory. Only those parameters which need to be changed for a dataset retrieval needs to be set in a <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> file!</p>
389<p>The limitation of a dataset to be retrieved should be done very cautiously. The datasets can differ in many ways and vary over the time in resolution and parameterisations methods, especially the operational model cycles improves through a lot of changes over the time. If you are not familiar with the data it might be useful or necessary to check for availability of data in ECMWF’s MARS:</p>
390<blockquote>
391<div><ul class="simple">
392<li><strong>Public users</strong> can use a web mask to check on data or list available data at this <a class="reference external" href="https://apps.ecmwf.int/datasets/">Public datasets web interface</a>.</li>
393<li><strong>Member state users</strong> can check availability of data online in the <a class="reference external" href="https://apps.ecmwf.int/mars-catalogue/">MARS catalogue</a>.</li>
394</ul>
395</div></blockquote>
396<p>There you can select step by step what data suits your needs. This would be the most straightforeward way of checking for available data and therefore limit the possibility of <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code> to fail. The following figure gives an example how the web interface would look like:</p>
397<div class="figure" id="ref-fig-mars-catalogue-ss">
398<img alt="_images/MARS_catalogue_snapshot.png" src="_images/MARS_catalogue_snapshot.png" />
399</div>
400<p>!!!!!!!! ADD HERE ANOTHER SCREENSHOT OF THE PARAMETER SELECTION AREA ( HAS TO BE DONE AT HOME )</p>
401<p>Additionally, you can find a lot of helpful links to dataset documentations, direct links to specific dataset web catalogues or further general information at the <a class="reference external" href="Ecmwf/ec-links.html">link collection</a> in the ECMWF data section.</p>
402<p><code class="docutils literal notranslate"><span class="pre">Flex_extract</span></code> is specialised to retrieve a limited number of datasets, namely <em>ERA-Interim</em>, <em>CERA-20C</em>, <em>ERA5</em> and <em>HRES (operational data)</em> as well as the <em>ENS (operational data, 15-day forecast)</em>. The limitation relates mainly to the dataset itself, the stream (what kind of forecast or what subset of dataset) and the experiment number. Mostly, the experiment number is equal to <code class="docutils literal notranslate"><span class="pre">1</span></code> to signal that the actual version should be used.</p>
403<p>The next level of differentiation would be the field type, level type and time period. <code class="docutils literal notranslate"><span class="pre">Flex_extract</span></code> currently only supports the main streams for the re-analysis datasets and provides extraction of different streams for the operational dataset. The possibilities of compositions of dataset and stream selection are represented by the current list of example <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> files. You can see this in the naming of the example files:</p>
404<div class="literal-block-wrapper docutils container" id="id4">
405<div class="code-block-caption"><span class="caption-text">“Current example <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> files distributed with <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code>. “</span><a class="headerlink" href="#id4" title="Permalink to this code"></a></div>
406<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>CONTROL_CERA
407CONTROL_CERA.global
408CONTROL_CERA.public
409CONTROL_EA5
410CONTROL_EA5.global
411CONTROL_EI
412CONTROL_EI.global
413CONTROL_EI.public
414CONTROL_OD.ELDA.FC.eta.ens.double
415CONTROL_OD.ENFO.CF
416CONTROL_OD.ENFO.CV
417CONTROL_OD.ENFO.PF
418CONTROL_OD.ENFO.PF.36hours
419CONTROL_OD.ENFO.PF.ens
420CONTROL_OD.OPER.4V.operational
421CONTROL_OD.OPER.FC.36hours
422CONTROL_OD.OPER.FC.eta.global
423CONTROL_OD.OPER.FC.eta.highres
424CONTROL_OD.OPER.FC.gauss.highres
425CONTROL_OD.OPER.FC.operational
426CONTROL_OD.OPER.FC.twiceaday.1hourly
427CONTROL_OD.OPER.FC.twiceaday.3hourly
428</pre></div>
429</div>
430</div>
431<p>The main differences and features in the datasets are listed in the table shown below:</p>
432<div class="figure" id="id5">
433<span id="ref-tab-dataset-cmp"></span><img alt="_images/dataset_cmp_table.png" src="_images/dataset_cmp_table.png" />
434<p class="caption"><span class="caption-text">DO THIS TABLE AGAIN BY HAND!</span></p>
435</div>
436<p>A common problem for beginners in retrieving ECMWF datasets is the mismatch in the definition of these parameters. For example, if you would like to retrieve operational data before <code class="docutils literal notranslate"><span class="pre">June</span> <span class="pre">25th</span> <span class="pre">2013</span></code> and set the maximum level to <code class="docutils literal notranslate"><span class="pre">137</span></code> you will get an error because this number of levels was first introduced at this effective day. So, be cautious in the combination of space and time resolution as well as the field types which are not available all the time.</p>
437<div class="admonition note">
438<p class="first admonition-title">Note</p>
439<p class="last">Sometimes it might not be clear how specific parameters in the control file must be set in terms of format. Please see the description of the parameters in section <a class="reference external" href="Documentation/Input/control_params.html">CONTROL parameters</a> or have a look at the ECMWF user documentation for <a class="reference external" href="https://confluence.ecmwf.int/display/UDOC/MARS+keywords">MARS keywords</a></p>
440</div>
441<p>In the following we shortly discuss the main retrieval opportunities of the different datasets and  categoize the <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> files.</p>
442<div class="section" id="public-datasets">
443<h3>Public datasets<a class="headerlink" href="#public-datasets" title="Permalink to this headline"></a></h3>
444<p>The main difference in the definition of a <code class="docutils literal notranslate"><span class="pre">CONRTOL</span></code> file for a public dataset is the setting of the parameter <code class="docutils literal notranslate"><span class="pre">DATASET</span></code>. This specification enables the selection of a public dataset in MARS. Otherwise the request would not find the dataset.
445For the two public datasets <em>CERA-20C</em> and <em>ERA-Interim</em> an example file with the ending <code class="docutils literal notranslate"><span class="pre">.public</span></code> is provided and can be used straightaway.</p>
446<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>CONTROL_CERA.public
447CONTROL_EI.public
448</pre></div>
449</div>
450<p>For <em>CERA-20C</em> it seems that there are no differences in the dataset against the full dataset, while the <em>public ERA-Interim</em> has only analysis fields every 6 hour without filling forecasts in between for model levels. Therefore it is only possible to retrieve 6-hourly data for <em>public ERA-Interim</em>.</p>
451<div class="admonition note">
452<p class="first admonition-title">Note</p>
453<p class="last">In general, <em>ERA5</em> is a public dataset. However, since the model levels are not yet publicly available, it is not possible to retrieve <em>ERA5</em> data to drive the <code class="docutils literal notranslate"><span class="pre">FLEXPART</span></code> model. As soon as this is possible it will be announced at the community website and per newsletter.</p>
454</div>
455</div>
456<div class="section" id="cera">
457<h3>CERA<a class="headerlink" href="#cera" title="Permalink to this headline"></a></h3>
458<p>For this dataset it is important to keep in mind that the dataset is available for the period 09/1901 until 12/2010 and the temporal resolution is limited to 3-hourly fields.
459It is also a pure ensemble data assimilation dataset and is stored under the <code class="docutils literal notranslate"><span class="pre">enda</span></code> stream. It has <code class="docutils literal notranslate"><span class="pre">10</span></code> ensemble members. The example <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> files will only select the first member (<code class="docutils literal notranslate"><span class="pre">number=0</span></code>). You may change this to another number or a list of numbers (e.g. <code class="docutils literal notranslate"><span class="pre">NUMBER</span> <span class="pre">0/to/10</span></code>).
460Another important difference to all other datasets is the forecast starting time which is 18 UTC. Which means that the forecast in <em>CERA-20C</em> for flux fields is  12 hours long. Since the forecast extends over a single day we need to extract one day in advance and one day subsequently. This is automatically done in <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code>.</p>
461</div>
462<div class="section" id="era-5">
463<h3>ERA 5<a class="headerlink" href="#era-5" title="Permalink to this headline"></a></h3>
464<p>This is the newest re-analysis dataset and has a temporal resolution of 1-hourly analysis fields. Up to date it is available until April 2019 with regular release of new months.
465The original horizontal resolution is <code class="docutils literal notranslate"><span class="pre">0.28125°</span></code> which needs some caution in the definition of the domain, since the length of the domain in longitude or latitude direction  must be an exact multiple of the resolution. It might be easier for users to use <code class="docutils literal notranslate"><span class="pre">0.25</span></code> for the resolution which MARS will automatically interpolate.
466The forecast starting time is <code class="docutils literal notranslate"><span class="pre">06/18</span> <span class="pre">UTC</span></code> which is important for the flux data. This should be set in the <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> file via the <code class="docutils literal notranslate"><span class="pre">ACCTIME</span> <span class="pre">06/18</span></code> parameter in correspondence with <code class="docutils literal notranslate"><span class="pre">ACCMAXSTEP</span> <span class="pre">12</span></code> and <code class="docutils literal notranslate"><span class="pre">ACCTYPE</span> <span class="pre">FC</span></code>.</p>
467<div class="admonition note">
468<p class="first admonition-title">Note</p>
469<p class="last">We know that <em>ERA5</em> also has an ensemble data assimilation system but this is not yet retrievable with <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code> since the deaccumulation of the flux fields works differently in this stream. Ensemble retrieval for <em>ERA5</em> is a future ToDo.</p>
470</div>
471</div>
472<div class="section" id="era-interim">
473<h3>ERA-Interim<a class="headerlink" href="#era-interim" title="Permalink to this headline"></a></h3>
474<p>This re-analysis dataset will exceed its end of production at 31st August 2019!
475It is then available from 1st January 1979 to 31st August 2019. The <code class="docutils literal notranslate"><span class="pre">etadot</span></code> is not available in this dataset. Therefore <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code> must select the <code class="docutils literal notranslate"><span class="pre">GAUSS</span></code> parameter to retrieve the divergence field in addition. The vertical velocity is the calculated with the continuity equation in the Fortran program <code class="docutils literal notranslate"><span class="pre">CONVERT2</span></code>. Since the analysis fields are only available for every 6th hour, the dataset can be made 3 hourly by adding forecast fields in between. No ensemble members are available.</p>
476<div class="admonition-todo admonition" id="index-0">
477<p class="first admonition-title">Todo</p>
478<p class="last">&#64;LEO: please check the complete description and functionality of the CONTROL FILEs</p>
479</div>
480</div>
481<div class="section" id="operational-data">
482<h3>Operational data<a class="headerlink" href="#operational-data" title="Permalink to this headline"></a></h3>
483<p>This is the real time atmospheric model in high resolution with a 10-day forecast. This means it underwent regular adaptations and improvements over the years. Hence, retrieving data from this dataset needs extra attention in selecting correct settings of parameter. See <a class="reference internal" href="#ref-tab-dataset-cmp"><span class="std std-ref">DO THIS TABLE AGAIN BY HAND!</span></a> for the most important parameters.
484Nowadays, it is available 1 hourly by filling the gaps of the 6 hourly analysis fields with 1 hourly forecast fields. Since 4th June 2008 the eta coordinate is directly available so that <code class="docutils literal notranslate"><span class="pre">ETA</span></code> should be set to <code class="docutils literal notranslate"><span class="pre">1</span></code> to save computation time. The horizontal resolution can be up to <code class="docutils literal notranslate"><span class="pre">0.1°</span></code> and in combination with <code class="docutils literal notranslate"><span class="pre">137</span></code> vertical levels can lead to troubles in retrieving this high resolution dataset in terms of job duration and quota exceedence.
485It is recommended to submit such high resolution cases for single day retrievals (see <code class="docutils literal notranslate"><span class="pre">JOB_CHUNK</span></code> parameter in <code class="docutils literal notranslate"><span class="pre">run.sh</span></code> script) to avoid job failures due to exceeding limits.</p>
486<p><code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> files for normal daily retrievals with a mix of analysis and forecast fields are listed below:</p>
487<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>CONTROL_OD.OPER.4V.eta.global
488CONTROL_OD.OPER.FC.eta.global
489CONTROL_OD.OPER.FC.eta.highres
490CONTROL_OD.OPER.FC.gauss.highres
491</pre></div>
492</div>
493<p>These files defines the minimum number of parameters necessary to retrieve a daily subset. The setup of field types is optimal and should only be changed if the user understands what he does. The grid, domain and temporal resolution can be changed according to availability.</p>
494<div class="admonition-todo admonition" id="index-1">
495<p class="first admonition-title">Todo</p>
496<p class="last">&#64;LEO - explain the setup with 4V fields! Who can extract it, when would this be useful?</p>
497</div>
498<div class="admonition note">
499<p class="first admonition-title">Note</p>
500<p class="last">Please see <a class="reference external" href="https://confluence.ecmwf.int/display/UDOC/Retrieve#Retrieve-Retrievalefficiency">Information about MARS retrievement</a> to get to know hints about retrieval efficiency and troubleshooting.</p>
501</div>
502<dl class="docutils">
503<dt>Pure forecast</dt>
504<dd><p class="first">It is possible to retrieve pure forecasts exceeding a day. The forecast period available depends on the date and forecast field type. Please use MARS catalogue to check the availability. Below are some examples for 36 hour forecast of <em>Forecast (FC)</em>, <em>Control forecast (CF)</em> and <em>Calibration/Validation forecast (CV)</em>.
505The <em>CV</em> field type was only available 3-hourly from 2006 up to 2016. It is recommended to use the <em>CF</em> type since this is available from 1992 (3-hourly) on up to today in 1-hourly temporal resolution. <em>CV</em> and <em>CF</em> field types belong to the <em>Ensemble prediction system (ENFO)</em> which contain 50 ensemble members.
506Please be aware that in this case it is necessary to set the specific type for flux fields explicitly, otherwise it could select a default value which might be different from what you expect!</p>
507<div class="last highlight-bash notranslate"><div class="highlight"><pre><span></span>CONTROL_OD.ENFO.CF.36hours
508CONTROL_OD.ENFO.CV.36hours
509CONTROL_OD.OPER.FC.36hours
510</pre></div>
511</div>
512</dd>
513<dt>Half-day retrievals</dt>
514<dd><p class="first">If a forecast for just half a day is wanted it can be done by substituting the analysis fields also by forecast fields as shown in files with <code class="docutils literal notranslate"><span class="pre">twiceaday</span></code> in it. They produce a full day retrieval with pure 12 hour forecasts twice a day. It is also possible to use the operational version which would get the time information from ECMWF’s environmental variables and therefore get the newest forecast per day. This version uses a <code class="docutils literal notranslate"><span class="pre">BASETIME</span></code> parameter which tells MARS to extract the exact 12 hours upfront to the selected date. If the <code class="docutils literal notranslate"><span class="pre">CONTROL</span></code> file with <code class="docutils literal notranslate"><span class="pre">basetime</span></code> in the filename is used this can be done for any other date too.</p>
515<div class="last highlight-bash notranslate"><div class="highlight"><pre><span></span>CONTROL_OD.OPER.FC.eta.basetime
516CONTROL_OD.OPER.FC.operational
517CONTROL_OD.OPER.FC.twiceaday.1hourly
518CONTROL_OD.OPER.FC.twiceaday.3hourly
519</pre></div>
520</div>
521</dd>
522<dt>Ensemble members</dt>
523<dd><p class="first">The retrieval of ensemble members were already mentioned in the pure forecast section and for <em>CERA-20C</em> data.
524In this <code class="docutils literal notranslate"><span class="pre">flex_extract</span></code> version there is an additional possibility to retrieve the <em>Ensemble Long window Data Assimilation (ELDA)</em> stream from the real-time dataset. This model version has (up to May 2019) 25 ensemble members and a control run (<code class="docutils literal notranslate"><span class="pre">number</span> <span class="pre">0</span></code>). Starting from June 2019 it has 50 ensemble members. Therefore we created the possibility to double up the 25 ensemble members (before June 2019) to 50 members by taking the original 25 members from MARS and subtracting 2 times the difference between the member value and the control value. This is done by selecting the parameter <code class="docutils literal notranslate"><span class="pre">DOUBLEELDA</span></code> and set it to <code class="docutils literal notranslate"><span class="pre">1</span></code>.</p>
525<div class="last highlight-bash notranslate"><div class="highlight"><pre><span></span>CONTROL_OD.ELDA.FC.eta.ens.double
526CONTROL_OD.ENFO.PF.ens
527</pre></div>
528</div>
529</dd>
530</dl>
531<div class="admonition-todo admonition" id="index-2">
532<p class="first admonition-title">Todo</p>
533<p class="last">&#64;LEO: Please tell me why perturbed forecast are possible? Is it because of some user rights? I have no opportunity of retrieve PF data.</p>
534</div>
535</div>
536<div class="section" id="specific-features">
537<h3>Specific features<a class="headerlink" href="#specific-features" title="Permalink to this headline"></a></h3>
538<dl class="docutils">
539<dt>rrint</dt>
540<dd>Decides if the precipitation flux data uses the old (<code class="docutils literal notranslate"><span class="pre">0</span></code>) or new (<code class="docutils literal notranslate"><span class="pre">1</span></code>) disaggregation scheme. See <a class="reference internal" href="Documentation/disagg.html"><span class="doc">Disaggregation of Flux Data</span></a> for explanaition.</dd>
541<dt>cwc</dt>
542<dd>Decides if the total cloud water content will be retrieved (set to <code class="docutils literal notranslate"><span class="pre">1</span></code>) in addition. This is the sum of cloud liquid and cloud ice water content.</dd>
543<dt>addpar</dt>
544<dd>With this parameter an additional list of 2-dimensional, non-flux parameters can be retrieved. Use format <code class="docutils literal notranslate"><span class="pre">param1/param2/.../paramx</span></code> to list the parameters. Please be consistent in using either the parameter IDs or the short names.</dd>
545<dt>doubleelda</dt>
546<dd>Use this to double the ensemble member number by adding further disturbance to each member.</dd>
547<dt>debug</dt>
548<dd>If set to <code class="docutils literal notranslate"><span class="pre">1</span></code> all temporary files were kept at the end. Otherwise everything except the final output files will be deleted.</dd>
549<dt>request</dt>
550<dd>This produces an extra <em>csv</em> file <code class="docutils literal notranslate"><span class="pre">mars_requests.csv</span></code> where the content of each mars request of the job is stored. Useful for debugging and documentation.</dd>
551<dt>mailfail</dt>
552<dd>At default the mail is send to the mail connected with the user account. Add additional email addresses if you want. But as soon as you enter a new mail, the default will be overwritten. If you would like to keep the mail from your user account, please add <code class="docutils literal notranslate"><span class="pre">${USER}</span></code> to the list ( comma seperated ) or mail addresses.</dd>
553</dl>
554</div>
555<div class="section" id="hints-for-definition-of-some-parameter-combinations">
556<h3>Hints for definition of some parameter combinations<a class="headerlink" href="#hints-for-definition-of-some-parameter-combinations" title="Permalink to this headline"></a></h3>
557<dl class="docutils">
558<dt>Field types and times</dt>
559<dd><p class="first">This combination is very important. It defines the temporal resolution and which field type is extracted per time step.
560The time declaration for analysis (AN) fields uses the times of the specific analysis and (forecast time) steps have to be <code class="docutils literal notranslate"><span class="pre">0</span></code>. The forecast field types (e.g. FC, CF, CV, PF) need to declare a combination of (forescast start) times and the (forecast) steps. Both of them together defines the actual time step. It is important to know the forecast starting times for the dataset to be retrieved, since they are different. In general it is enough to give information for the exact time steps, but it is also possible to have more time step combinations of <code class="docutils literal notranslate"><span class="pre">TYPE</span></code>, <code class="docutils literal notranslate"><span class="pre">TIME</span></code> and <code class="docutils literal notranslate"><span class="pre">STEP</span></code> because the temporal (hourly) resolution with the <code class="docutils literal notranslate"><span class="pre">DTIME</span></code> parameter will select the correct combinations.</p>
561<div class="literal-block-wrapper last docutils container" id="id6">
562<div class="code-block-caption"><span class="caption-text">Example of a setting for the field types and temporal resolution.</span><a class="headerlink" href="#id6" title="Permalink to this code"></a></div>
563<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span> DTIME <span class="m">3</span>
564 TYPE AN FC FC FC AN FC FC FC
565 TIME <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">00</span> <span class="m">12</span> <span class="m">12</span> <span class="m">12</span> <span class="m">12</span>
566 STEP <span class="m">00</span> <span class="m">03</span> <span class="m">06</span> <span class="m">09</span> <span class="m">00</span> <span class="m">03</span> <span class="m">06</span> <span class="m">09</span>
567</pre></div>
568</div>
569</div>
570</dd>
571<dt>Vertical velocity</dt>
572<dd><p class="first">The vertical velocity for <code class="docutils literal notranslate"><span class="pre">FLEXPART</span></code> is not directly available from MARS. Therefore it has to be calculated. There are a couple of different options. The following parameters are responsible for the selection. See <a class="reference internal" href="Documentation/vertco.html"><span class="doc">Vertical Coordinate</span></a> for a detailed explanation. The <code class="docutils literal notranslate"><span class="pre">ETADIFF</span></code>, <code class="docutils literal notranslate"><span class="pre">OMEGA</span></code> and <code class="docutils literal notranslate"><span class="pre">OMEGADIFF</span></code> versions are only recommended for debugging and testing reasons. Usually it is a decision between <code class="docutils literal notranslate"><span class="pre">GAUSS</span></code> and <code class="docutils literal notranslate"><span class="pre">ETA</span></code>, where for <code class="docutils literal notranslate"><span class="pre">GAUSS</span></code> spectral fields of the horizontal wind fields and the divergence are to be retrieved and used with the continuity equation to calculate the vertical velocity. For <code class="docutils literal notranslate"><span class="pre">ETA</span></code> the latitude/longitude fields of horizontal wind fields and eta-coordinate are to be retrieved. It is recommended to use <code class="docutils literal notranslate"><span class="pre">ETA</span></code> where possible due to a reduced computation time.</p>
573<div class="literal-block-wrapper last docutils container" id="id7">
574<div class="code-block-caption"><span class="caption-text">Example setting for the vertical coordinate retrieval.</span><a class="headerlink" href="#id7" title="Permalink to this code"></a></div>
575<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>GAUSS <span class="m">0</span>
576ETA <span class="m">1</span>
577ETADIFF <span class="m">0</span>
578DPDETA <span class="m">1</span>
579OMEGA <span class="m">0</span>
580OMEGADIFF <span class="m">0</span>
581</pre></div>
582</div>
583</div>
584</dd>
585<dt>Grid resolution and domain</dt>
586<dd><p class="first">The grid and domain selection depends on each other. The grid can be defined in the format of normal degrees (e.g. <code class="docutils literal notranslate"><span class="pre">1.</span></code>) or as in older versions by 1/1000. degrees (e.g. <code class="docutils literal notranslate"><span class="pre">1000</span></code> for <code class="docutils literal notranslate"><span class="pre">1°</span></code>).
587After selecting the grid, the domain has to be defined in a way that the length of the domain in longitude or latitude direction  must be an exact multiple of the grid.
588The horizontal resolution for spectral fields will be set by the parameter <code class="docutils literal notranslate"><span class="pre">RESOL</span></code>. For information about how to select an appropriate value you can read the explanation of the MARS keyword <a class="reference external" href="https://confluence.ecmwf.int/display/UDOC/Post-processing+keywords#Post-processingkeywords-resol">here</a> and in <a class="reference external" href="https://confluence.ecmwf.int/display/UDOC/Retrieve#Retrieve-Truncationbeforeinterpolation">this table</a>.</p>
589<div class="literal-block-wrapper last docutils container" id="id8">
590<div class="code-block-caption"><span class="caption-text">Example setting for a northern hemisphere domain with a grid of <code class="docutils literal notranslate"><span class="pre">0.25°</span></code>.</span><a class="headerlink" href="#id8" title="Permalink to this code"></a></div>
591<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>GRID <span class="m">0</span>.25
592RESOL <span class="m">799</span>
593SMOOTH <span class="m">0</span>
594UPPER <span class="m">90</span>.
595LOWER <span class="m">0</span>.
596LEFT -179.75
597RIGHT <span class="m">180</span>.
598</pre></div>
599</div>
600</div>
601</dd>
602<dt>Flux data</dt>
603<dd><p class="first">The flux fields are accumulated forecast fields all the time. Since some re-analysis dataset nowadays have complete set of analysis fields in their temporal resolution it was important to define a new parameter set to define the flux fields since the information could not be taken from <code class="docutils literal notranslate"><span class="pre">TYPE</span></code>, <code class="docutils literal notranslate"><span class="pre">TIME</span></code> and <code class="docutils literal notranslate"><span class="pre">STEP</span></code> any longer. Select a forecast field type <code class="docutils literal notranslate"><span class="pre">ACCTYPE</span></code>, the forecast starting time <code class="docutils literal notranslate"><span class="pre">ACCTIME</span></code> and the maximum forecast step <code class="docutils literal notranslate"><span class="pre">ACCMAXSTEP</span></code>. The <code class="docutils literal notranslate"><span class="pre">DTIME</span></code> parameter defines the temporal resolution for the whole period.</p>
604<div class="literal-block-wrapper last docutils container" id="id9">
605<div class="code-block-caption"><span class="caption-text">Example setting for the definition of flux fields.</span><a class="headerlink" href="#id9" title="Permalink to this code"></a></div>
606<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span> DTIME <span class="m">3</span>
607 ACCTYPE FC
608 ACCTIME <span class="m">00</span>/12
609 ACCMAXSTEP <span class="m">36</span>
610</pre></div>
611</div>
612</div>
613</dd>
614</dl>
615<div class="toctree-wrapper compound">
616</div>
617</div>
618</div>
619</div>
620
621
622           </div>
623           
624          </div>
625          <footer>
626 
627    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
628     
629        <a href="ecmwf_data.html" class="btn btn-neutral float-right" title="ECMWF Data" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
630     
631     
632        <a href="Installation/local.html" class="btn btn-neutral float-left" title="Local mode installation" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
633     
634    </div>
635 
636
637  <hr/>
638
639  <div role="contentinfo">
640    <p>
641        &copy; Copyright 2019, Anne Philipp and Leopold Haimberger
642
643    </p>
644  </div>
645  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
646
647</footer>
648
649        </div>
650      </div>
651
652    </section>
653
654  </div>
655 
656
657
658  <script type="text/javascript">
659      jQuery(function () {
660          SphinxRtdTheme.Navigation.enable(true);
661      });
662  </script>
663
664 
665 
666   
667   
668
669</body>
670</html>
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG