Index: documentation/Sphinx/build/html/_modules/get_mars_data.html
===================================================================
--- documentation/Sphinx/build/html/_modules/get_mars_data.html	(revision 2625ca8faac35d65699e83e37bb42cdc24163490)
+++ documentation/Sphinx/build/html/_modules/get_mars_data.html	(revision f66f9905a36e7b0a824c539e948b15ca55615e47)
@@ -220,5 +220,5 @@
     <span class="n">inspect</span><span class="o">.</span><span class="n">getfile</span><span class="p">(</span><span class="n">inspect</span><span class="o">.</span><span class="n">currentframe</span><span class="p">())))</span> <span class="o">+</span> <span class="s1">&#39;/../&#39;</span><span class="p">)</span>
 <span class="kn">import</span> <span class="nn">_config</span>
-<span class="kn">from</span> <span class="nn">tools</span> <span class="k">import</span> <span class="p">(</span><span class="n">my_error</span><span class="p">,</span> <span class="n">normal_exit</span><span class="p">,</span> <span class="n">get_cmdline_args</span><span class="p">,</span>
+<span class="kn">from</span> <span class="nn">tools</span> <span class="k">import</span> <span class="p">(</span><span class="n">setup_controldata</span><span class="p">,</span> <span class="n">my_error</span><span class="p">,</span> <span class="n">normal_exit</span><span class="p">,</span> <span class="n">get_cmdline_args</span><span class="p">,</span>
                    <span class="n">read_ecenv</span><span class="p">,</span> <span class="n">make_dir</span><span class="p">)</span>
 <span class="kn">from</span> <span class="nn">classes.EcFlexpart</span> <span class="k">import</span> <span class="n">EcFlexpart</span>
@@ -227,8 +227,14 @@
 
 <span class="k">try</span><span class="p">:</span>
-    <span class="n">ecapi</span> <span class="o">=</span> <span class="kc">True</span>
+    <span class="n">ec_api</span> <span class="o">=</span> <span class="kc">True</span>
     <span class="kn">import</span> <span class="nn">ecmwfapi</span>
 <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
-    <span class="n">ecapi</span> <span class="o">=</span> <span class="kc">False</span>
+    <span class="n">ec_api</span> <span class="o">=</span> <span class="kc">False</span>
+
+<span class="k">try</span><span class="p">:</span>
+    <span class="n">cds_api</span> <span class="o">=</span> <span class="kc">True</span>
+    <span class="kn">import</span> <span class="nn">cdsapi</span>
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+    <span class="n">cds_api</span> <span class="o">=</span> <span class="kc">False</span>
 <span class="c1"># ------------------------------------------------------------------------------</span>
 <span class="c1"># FUNCTION</span>
@@ -248,14 +254,7 @@
 <span class="sd">    &#39;&#39;&#39;</span>
 
-    <span class="n">args</span> <span class="o">=</span> <span class="n">get_cmdline_args</span><span class="p">()</span>
-    <span class="n">c</span> <span class="o">=</span> <span class="n">ControlFile</span><span class="p">(</span><span class="n">args</span><span class="o">.</span><span class="n">controlfile</span><span class="p">)</span>
-
-    <span class="n">env_parameter</span> <span class="o">=</span> <span class="n">read_ecenv</span><span class="p">(</span><span class="n">_config</span><span class="o">.</span><span class="n">PATH_ECMWF_ENV</span><span class="p">)</span>
-    <span class="n">c</span><span class="o">.</span><span class="n">assign_args_to_control</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
-    <span class="n">c</span><span class="o">.</span><span class="n">assign_envs_to_control</span><span class="p">(</span><span class="n">env_parameter</span><span class="p">)</span>
-    <span class="n">c</span><span class="o">.</span><span class="n">check_conditions</span><span class="p">(</span><span class="n">args</span><span class="o">.</span><span class="n">queue</span><span class="p">)</span>
-
+    <span class="n">c</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">setup_controldata</span><span class="p">()</span>
     <span class="n">get_mars_data</span><span class="p">(</span><span class="n">c</span><span class="p">)</span>
-    <span class="n">normal_exit</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">mailops</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">queue</span><span class="p">,</span> <span class="s1">&#39;Done!&#39;</span><span class="p">)</span>
+    <span class="n">normal_exit</span><span class="p">(</span><span class="s1">&#39;Retrieving MARS data: Done!&#39;</span><span class="p">)</span>
 
     <span class="k">return</span></div>
@@ -278,5 +277,6 @@
 
 <span class="sd">    &#39;&#39;&#39;</span>
-    <span class="n">c</span><span class="o">.</span><span class="n">ecapi</span> <span class="o">=</span> <span class="n">ecapi</span>
+    <span class="n">c</span><span class="o">.</span><span class="n">ec_api</span> <span class="o">=</span> <span class="n">ec_api</span>
+    <span class="n">c</span><span class="o">.</span><span class="n">cds_api</span> <span class="o">=</span> <span class="n">cds_api</span>
 
     <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">inputdir</span><span class="p">):</span>
@@ -335,5 +335,9 @@
 
 <div class="viewcode-block" id="mk_server"><a class="viewcode-back" href="../api.html#get_mars_data.mk_server">[docs]</a><span class="k">def</span> <span class="nf">mk_server</span><span class="p">(</span><span class="n">c</span><span class="p">):</span>
-    <span class="sd">&#39;&#39;&#39;Creates server connection if ECMWF WebAPI is available.</span>
+    <span class="sd">&#39;&#39;&#39;Creates a server connection with available python API.</span>
+
+<span class="sd">    Which API is used depends on availability and the dataset to be retrieved.</span>
+<span class="sd">    The CDS API is used for ERA5 dataset no matter if the user is a member or</span>
+<span class="sd">    a public user. ECMWF WebAPI is used for all other available datasets.</span>
 
 <span class="sd">    Parameters</span>
@@ -345,19 +349,33 @@
 <span class="sd">    Return</span>
 <span class="sd">    ------</span>
-<span class="sd">    server : ECMWFDataServer or ECMWFService</span>
-<span class="sd">        Connection to ECMWF server via python interface ECMWF WebAPI.</span>
-
-<span class="sd">    &#39;&#39;&#39;</span>
-    <span class="k">if</span> <span class="n">c</span><span class="o">.</span><span class="n">ecapi</span><span class="p">:</span>
+<span class="sd">    server : ECMWFDataServer, ECMWFService or Client</span>
+<span class="sd">        Connection to ECMWF server via python interface ECMWF WebAPI or CDS API.</span>
+
+<span class="sd">    &#39;&#39;&#39;</span>
+    <span class="k">if</span> <span class="n">cds_api</span> <span class="ow">and</span> <span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">marsclass</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;EA&#39;</span><span class="p">):</span>
+        <span class="n">server</span> <span class="o">=</span> <span class="n">cdsapi</span><span class="o">.</span><span class="n">Client</span><span class="p">()</span>
+        <span class="n">c</span><span class="o">.</span><span class="n">ec_api</span> <span class="o">=</span> <span class="kc">False</span>
+    <span class="k">elif</span> <span class="n">c</span><span class="o">.</span><span class="n">ec_api</span><span class="p">:</span>
         <span class="k">if</span> <span class="n">c</span><span class="o">.</span><span class="n">public</span><span class="p">:</span>
             <span class="n">server</span> <span class="o">=</span> <span class="n">ecmwfapi</span><span class="o">.</span><span class="n">ECMWFDataServer</span><span class="p">()</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="n">server</span> <span class="o">=</span> <span class="n">ecmwfapi</span><span class="o">.</span><span class="n">ECMWFService</span><span class="p">(</span><span class="s2">&quot;mars&quot;</span><span class="p">)</span>
+        <span class="n">c</span><span class="o">.</span><span class="n">cds_api</span> <span class="o">=</span> <span class="kc">False</span>
     <span class="k">else</span><span class="p">:</span>
         <span class="n">server</span> <span class="o">=</span> <span class="kc">False</span>
 
-    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Using ECMWF WebAPI: &#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">ecapi</span><span class="p">))</span>
+    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Using ECMWF WebAPI: &#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">ec_api</span><span class="p">))</span>
+    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Using CDS API: &#39;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">cds_api</span><span class="p">))</span>
 
     <span class="k">return</span> <span class="n">server</span></div>
+
+
+<div class="viewcode-block" id="check_dates_for_nonflux_fc_times"><a class="viewcode-back" href="../api.html#get_mars_data.check_dates_for_nonflux_fc_times">[docs]</a><span class="k">def</span> <span class="nf">check_dates_for_nonflux_fc_times</span><span class="p">(</span><span class="n">types</span><span class="p">,</span> <span class="n">times</span><span class="p">):</span>
+    <span class="sd">&#39;&#39;&#39;</span>
+<span class="sd">    &#39;&#39;&#39;</span>
+    <span class="k">for</span> <span class="n">ty</span><span class="p">,</span> <span class="n">ti</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">types</span><span class="p">,</span><span class="n">times</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">ty</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;FC&#39;</span> <span class="ow">and</span> <span class="nb">int</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span> <span class="o">==</span> <span class="mi">18</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">True</span>
+    <span class="k">return</span> <span class="kc">False</span></div>
 
 
@@ -405,11 +423,15 @@
     <span class="n">chunk</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">date_chunk</span><span class="p">))</span>
 
-    <span class="k">if</span> <span class="n">c</span><span class="o">.</span><span class="n">basetime</span><span class="p">:</span>
-        <span class="k">if</span> <span class="n">c</span><span class="o">.</span><span class="n">basetime</span> <span class="o">==</span> <span class="s1">&#39;00&#39;</span><span class="p">:</span>
-            <span class="n">start</span> <span class="o">=</span> <span class="n">start</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
-
-    <span class="k">if</span> <span class="ow">not</span> <span class="n">c</span><span class="o">.</span><span class="n">purefc</span> <span class="ow">and</span> <span class="n">fluxes</span><span class="p">:</span>
+    <span class="k">if</span> <span class="n">c</span><span class="o">.</span><span class="n">basetime</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="n">start</span> <span class="o">=</span> <span class="n">start</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">c</span><span class="o">.</span><span class="n">purefc</span> <span class="ow">and</span> <span class="n">fluxes</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">c</span><span class="o">.</span><span class="n">basetime</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
         <span class="n">start</span> <span class="o">=</span> <span class="n">start</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
         <span class="n">end</span> <span class="o">=</span> <span class="n">end</span> <span class="o">+</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+
+    <span class="c1"># if we have non-flux forecast data starting at 18 UTC</span>
+    <span class="c1"># we need to start retrieving data one day in advance</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">fluxes</span> <span class="ow">and</span> <span class="n">check_dates_for_nonflux_fc_times</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">type</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">time</span><span class="p">):</span>
+        <span class="n">start</span> <span class="o">=</span> <span class="n">start</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
 
     <span class="k">return</span> <span class="n">start</span><span class="p">,</span> <span class="n">end</span><span class="p">,</span> <span class="n">chunk</span></div>
@@ -493,5 +515,5 @@
             <span class="n">flexpart</span><span class="o">.</span><span class="n">retrieve</span><span class="p">(</span><span class="n">server</span><span class="p">,</span> <span class="n">dates</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">public</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">request</span><span class="p">,</span> <span class="n">c</span><span class="o">.</span><span class="n">inputdir</span><span class="p">)</span>
         <span class="k">except</span> <span class="ne">IOError</span><span class="p">:</span>
-            <span class="n">my_error</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">mailfail</span><span class="p">,</span> <span class="s1">&#39;MARS request failed&#39;</span><span class="p">)</span>
+            <span class="n">my_error</span><span class="p">(</span><span class="s1">&#39;MARS request failed&#39;</span><span class="p">)</span>
 
         <span class="n">day</span> <span class="o">+=</span> <span class="n">delta_t</span>
@@ -513,5 +535,5 @@
   <div role="contentinfo">
     <p>
-        &copy; Copyright 2018, Anne Philipp and Leopold Haimberger
+        &copy; Copyright 2018, Anne Philipp and Leopold Haimberger.
 
     </p>
@@ -532,11 +554,19 @@
   
 
-    
-    
-      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
-        <script type="text/javascript" src="../_static/jquery.js"></script>
-        <script type="text/javascript" src="../_static/underscore.js"></script>
-        <script type="text/javascript" src="../_static/doctools.js"></script>
-    
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../',
+            VERSION:'7.1 alpha',
+            LANGUAGE:'None',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../_static/jquery.js"></script>
+      <script type="text/javascript" src="../_static/underscore.js"></script>
+      <script type="text/javascript" src="../_static/doctools.js"></script>
+      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
 
   
