source: flexpart.git/flexpart_code/checkgrib/test/checkgrib_test.py @ 9411952

FPv9.3.2
Last change on this file since 9411952 was 9411952, checked in by Don Morton <Don.Morton@…>, 3 years ago

Synchronised repo with the current CTBTO master.
The sole change is the addition of the flexpart_code/checkgrib/ dir
  • Property mode set to 100755
File size: 53.6 KB
Line 
1#!/usr/bin/env python
2
3"""
4Don Morton
5Boreal Scientific Computing LLC
6Fairbanks, Alaska, USA
7Don.Morton@borealscicomp.com
8"""
9
10
11
12import argparse
13import fnmatch
14import logging
15import os
16import shutil
17import subprocess
18import sys
19import tempfile
20import uuid
21
22import numpy as np
23
24
25
26# Set up module logging
27LOGGING_LEVEL = logging.WARNING
28
29LOGGING_FORMAT = '%(levelname)s %(filename)s:%(funcName)s:%(lineno)d: %(message)s'
30
31MY_LOGGER = logging.getLogger()
32MY_LOGGER.setLevel(LOGGING_LEVEL)
33
34console_handler = logging.StreamHandler()
35console_handler.setLevel(LOGGING_LEVEL)
36
37logformatter = logging.Formatter(LOGGING_FORMAT)
38console_handler.setFormatter(logformatter)
39
40MY_LOGGER.addHandler(console_handler)
41#MY_LOGGER.propagate = False
42
43
44#---------------------  Define path prefixes to grib files  ------------
45#########   Paths to GRIB Files
46#### TO DO - put a note in here about where to get these files - put them
47#### on my web server.  Right now they're versioned, but that's almost
48###  1 GByte.
49ECMWF_PREFIX = 'gribfiles/ecmwf0p5'
50NCEP_PREFIX = 'gribfiles/ncep0p5'
51NCEPFV3_PREFIX = 'gribfiles/ncepfv30p5'
52
53
54
55### DJM - 2019-07-30 - using these paths will take a bit of work,
56### especially with the new ncep directory structure (with the 0.5/ and
57### 0.5.fv3/ subdirectories).  So, code will need to be adjusted in order
58### for this to work.
59#XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
60#If you're testing at CTBTO, these would be better paths to the same files
61#CTBTO_PREFIX = '/ops/data/atm'
62#ECMWF_PREFIX = CTBTO_PREFIX + '/ecmwf/2019/01/11/0.5'
63#NCEP_PREFIX = CTBTO_PREFIX + '/ncep/2019/01/08/0.5'
64#-----------------------------------------------------------------------
65
66
67
68
69# If you have these files available in the specified prefix, you should be
70# good to go   
71ECMWF0P5_GRIB2_FCST_TESTFILE = ECMWF_PREFIX + '/EN19011109'  #  forecast file
72ECMWF0P5_GRIB2_ANL_TESTFILE = ECMWF_PREFIX + '/EN19011112'   #  analysis file
73ECMWF0P5_EF_FCST_TESTFILE = ECMWF_PREFIX + '/EF19073103'  #  forecast file
74
75
76NCEP0P5_GRIB2_TESTFILE1 = NCEP_PREFIX + '/GD19010812'
77NCEP0P5_GRIB2_TESTFILE2 = NCEP_PREFIX + '/GD19010815'
78NCEP0P5_GRIB2_TESTFILE3 = NCEP_PREFIX + '/GD19010818'
79NCEP0P5_GRIB2_TESTFILE4 = NCEP_PREFIX + '/GD19010821'
80
81NCEPFV30P5_GRIB2_TESTFILE1 = NCEPFV3_PREFIX + '/GD19060800'
82NCEPFV30P5_GRIB2_TESTFILE2 = NCEPFV3_PREFIX + '/GD19060803'
83NCEPFV30P5_GRIB2_TESTFILE2 = NCEPFV3_PREFIX + '/GD19060806'
84
85def main():
86
87    # Currently set up to be in the parent directory.
88    CHECKGRIB_DIR = os.path.abspath(os.pardir)
89    CHECKGRIB = CHECKGRIB_DIR + '/checkGRIB'
90
91    # Compile flexpart2ctbt
92    print("Compiling checkGRIB in dir: " + str(CHECKGRIB_DIR))
93    compile_passed = test_compile_checkgrib(checkgrib_dir=CHECKGRIB_DIR)
94
95    print("compile_passed: %s" % compile_passed)
96
97    if not compile_passed:
98        print("Whooaa, Nellie, no sense continuing if you can't even compile!")
99        sys.exit()
100
101
102
103
104    passed_count = 0
105    failed_count = 0
106
107
108
109
110    #----------------------
111    # General tests
112    #----------------------
113
114    print("--------test_missing_source_cli_arg_detected--------")
115    passed = test_missing_source_cli_arg_detected(checkgrib=CHECKGRIB, 
116                           gribfile=ECMWF0P5_GRIB2_FCST_TESTFILE)
117    if passed: passed_count += 1
118    else: failed_count +=1 
119    print("passed: %s" % passed)
120    print("-----------------------------------------------")
121
122    print("--------test_invalid_source_cli_arg_detected--------")
123    passed = test_invalid_source_cli_arg_detected(checkgrib=CHECKGRIB, 
124                           gribfile=ECMWF0P5_GRIB2_FCST_TESTFILE)
125    if passed: passed_count += 1
126    else: failed_count +=1 
127    print("passed: %s" % passed)
128    print("-----------------------------------------------")
129
130    print("--------test_missing_levels_cli_arg_detected--------")
131    passed = test_missing_levels_cli_arg_detected(checkgrib=CHECKGRIB, 
132                           gribfile=ECMWF0P5_GRIB2_FCST_TESTFILE)
133    if passed: passed_count += 1
134    else: failed_count +=1 
135    print("passed: %s" % passed)
136    print("-----------------------------------------------")
137
138
139    print("--------test_checkanl_and_checkfcst_detected--------")
140    passed = test_checkanl_and_checkfcst_detected(checkgrib=CHECKGRIB, 
141                           gribfile=ECMWF0P5_GRIB2_FCST_TESTFILE)
142    if passed: passed_count += 1
143    else: failed_count +=1 
144    print("passed: %s" % passed)
145    print("-----------------------------------------------")
146
147
148    print("--------test_reports_no_cli_args--------")
149    passed = test_reports_no_cli_args(checkgrib=CHECKGRIB) 
150    if passed: passed_count += 1
151    else: failed_count +=1 
152    print("passed: %s" % passed)
153    print("-----------------------------------------------")
154
155
156    print("--------test_reports_no_path_args--------")
157    passed = test_reports_no_path_args(checkgrib=CHECKGRIB) 
158    if passed: passed_count += 1
159    else: failed_count +=1 
160    print("passed: %s" % passed)
161    print("-----------------------------------------------")
162
163
164
165
166
167
168
169    #----------------------
170    # ECMWF-specific tests
171    #----------------------
172
173    print("--------test_successful_ecmwfgrib2_check--------")
174    passed = test_successful_ecmwfgrib2_check(checkgrib=CHECKGRIB, 
175                                gribfile=ECMWF0P5_GRIB2_FCST_TESTFILE)
176    if passed: passed_count += 1
177    else: failed_count +=1 
178    print("passed: %s" % passed)
179    print("-----------------------------------------------")
180
181
182    print("--------test_successful_ecmwfgrib_ef_check--------")
183    passed = test_successful_ecmwfgrib_ef_check(checkgrib=CHECKGRIB, 
184                                ef_gribfile=ECMWF0P5_EF_FCST_TESTFILE)
185    if passed: passed_count += 1
186    else: failed_count +=1 
187    print("passed: %s" % passed)
188    print("-----------------------------------------------")
189
190
191    print("--------test_reports_ecmwfgrib_ef_fails_with_checkgrib2--------")
192    passed = test_reports_ecmwfgrib_ef_fails_with_checkgrib2(checkgrib=CHECKGRIB, 
193                                ef_gribfile=ECMWF0P5_EF_FCST_TESTFILE)
194    if passed: passed_count += 1
195    else: failed_count +=1 
196    print("passed: %s" % passed)
197    print("-----------------------------------------------")
198
199    print("--------test_reports_unexpected_ecmwf_grib_center--------")
200    passed = test_reports_unexpected_ecmwf_grib_centre(checkgrib=CHECKGRIB,
201                                   ncep_gribfile=NCEP0P5_GRIB2_TESTFILE1)
202    if passed: passed_count += 1
203    else: failed_count +=1 
204    print("passed: %s" % passed)
205    print("-----------------------------------------------")
206
207
208    print("--------test_reports_unexpected_ecmwf_level--------")
209    passed = test_reports_unexpected_ecmwf_level(checkgrib=CHECKGRIB, 
210                                   ecmwf_gribfile=ECMWF0P5_GRIB2_FCST_TESTFILE)
211    if passed: passed_count += 1
212    else: failed_count +=1 
213    print("passed: %s" % passed)
214    print("-----------------------------------------------")
215
216
217    print("--------test_reports_missing_ecmwf_levels--------")
218    passed = test_reports_missing_ecmwf_levels(checkgrib=CHECKGRIB, 
219                                   ecmwf_gribfile=ECMWF0P5_GRIB2_FCST_TESTFILE)
220    if passed: passed_count += 1
221    else: failed_count +=1 
222    print("passed: %s" % passed)
223    print("-----------------------------------------------")
224
225
226    print("--------test_reports_failed_ecmwf_anl_check--------")
227    passed = test_reports_failed_ecmwf_anl_check(checkgrib=CHECKGRIB, 
228                           ecmwf_fcst_gribfile=ECMWF0P5_GRIB2_FCST_TESTFILE)
229    if passed: passed_count += 1
230    else: failed_count +=1 
231    print("passed: %s" % passed)
232    print("-----------------------------------------------")
233
234
235    print("--------test_reports_failed_ecmwf_fcst_check--------")
236    passed = test_reports_failed_ecmwf_fcst_check(checkgrib=CHECKGRIB, 
237                           ecmwf_anl_gribfile=ECMWF0P5_GRIB2_ANL_TESTFILE)
238    if passed: passed_count += 1
239    else: failed_count +=1 
240    print("passed: %s" % passed)
241    print("-----------------------------------------------")
242
243
244    print("--------test_ecmwf_detects_missing_level_14--------")
245    passed = test_ecmwf_detects_missing_level_14(checkgrib=CHECKGRIB, 
246                           ecmwf_gribfile=ECMWF0P5_GRIB2_ANL_TESTFILE)
247    if passed: passed_count += 1
248    else: failed_count +=1 
249    print("passed: %s" % passed)
250    print("-----------------------------------------------")
251
252    print("--------test_ecmwf_detects_missing_level_14_etadot--------")
253    passed = test_ecmwf_detects_missing_level_14_etadot(checkgrib=CHECKGRIB, 
254                           ecmwf_gribfile=ECMWF0P5_GRIB2_ANL_TESTFILE)
255    if passed: passed_count += 1
256    else: failed_count +=1 
257    print("passed: %s" % passed)
258    print("-----------------------------------------------")
259
260
261    print("--------test_ecmwf_detects_missing_var_lsp--------")
262    passed = test_ecmwf_detects_missing_var_lsp(checkgrib=CHECKGRIB, 
263                           ecmwf_gribfile=ECMWF0P5_GRIB2_ANL_TESTFILE)
264    if passed: passed_count += 1
265    else: failed_count +=1 
266    print("passed: %s" % passed)
267    print("-----------------------------------------------")
268
269
270    print("--------test_ecmwf_detects_grib1_nsss--------")
271    passed = test_ecmwf_detects_grib1_nsss(checkgrib=CHECKGRIB, 
272                           ecmwf_gribfile=ECMWF0P5_GRIB2_ANL_TESTFILE)
273    if passed: passed_count += 1
274    else: failed_count +=1 
275    print("passed: %s" % passed)
276    print("-----------------------------------------------")
277
278
279
280    # The following two tests will never pass with current CTBTO ECMWF
281    # files because the files have mixed anl and fcst messages
282    """
283    print("--------test_ecmwf_successful_fcst_check--------")
284    passed = test_ecmwf_successful_fcst_check(checkgrib=CHECKGRIB,
285                           ecmwf_fcst_gribfile=ECMWF0P5_GRIB2_FCST_TESTFILE)
286    if passed: passed_count += 1
287    else: failed_count +=1
288    print("passed: %s" % passed)
289    print("-----------------------------------------------")
290
291
292    print("--------test_ecmwf_successful_anl_check--------")
293    passed = test_ecmwf_successful_anl_check(checkgrib=CHECKGRIB,
294                           ecmwf_anl_gribfile=ECMWF0P5_GRIB2_ANL_TESTFILE)
295    if passed: passed_count += 1
296    else: failed_count +=1
297    print("passed: %s" % passed)
298    print("-----------------------------------------------")
299    """
300
301    #----------------------
302    # NCEP-specific tests
303    #----------------------
304
305
306    print("--------test_reports_unexpected_ncep_grib_center--------")
307    passed = test_reports_unexpected_ncep_grib_centre(checkgrib=CHECKGRIB,
308                                   ecmwf_gribfile=ECMWF0P5_GRIB2_FCST_TESTFILE)
309    if passed: passed_count += 1
310    else: failed_count +=1 
311    print("passed: %s" % passed)
312    print("-----------------------------------------------")
313
314
315    print("--------test_successful_ncepgrib2_check--------")
316    passed = test_successful_ncepgrib2_check(checkgrib=CHECKGRIB, 
317                                gribfile=NCEP0P5_GRIB2_TESTFILE1)
318    if passed: passed_count += 1
319    else: failed_count +=1 
320    print("passed: %s" % passed)
321    print("-----------------------------------------------")
322
323
324    print("--------test_detects_ncep_more_pressure_levels_than_expected------")
325    passed = test_detects_ncep_more_pressure_levels_than_expected(
326                                checkgrib=CHECKGRIB, 
327                                gribfile=NCEP0P5_GRIB2_TESTFILE1)
328    if passed: passed_count += 1
329    else: failed_count +=1 
330    print("passed: %s" % passed)
331    print("-----------------------------------------------")
332
333    print("--------test_detects_ncep_fewer_pressure_levels_than_expected------")
334    passed = test_detects_ncep_fewer_pressure_levels_than_expected(
335                                checkgrib=CHECKGRIB, 
336                                gribfile=NCEP0P5_GRIB2_TESTFILE1)
337    if passed: passed_count += 1
338    else: failed_count +=1 
339    print("passed: %s" % passed)
340    print("-----------------------------------------------")
341
342
343    print("--------test_ncep_detects_missing_level_850------")
344    passed = test_ncep_detects_missing_level_850(
345                                checkgrib=CHECKGRIB, 
346                                ncep_gribfile=NCEP0P5_GRIB2_TESTFILE1)
347    if passed: passed_count += 1
348    else: failed_count +=1 
349    print("passed: %s" % passed)
350    print("-----------------------------------------------")
351
352
353    print("--------test_ncep_detects_missing_level_100_r------")
354    passed = test_ncep_detects_missing_level_100_r(
355                                checkgrib=CHECKGRIB, 
356                                ncep_gribfile=NCEP0P5_GRIB2_TESTFILE1)
357    if passed: passed_count += 1
358    else: failed_count +=1 
359    print("passed: %s" % passed)
360    print("-----------------------------------------------")
361
362
363    print("--------test_ncep_detects_missing_var_10u------")
364    passed = test_ncep_detects_missing_var_10u(
365                                checkgrib=CHECKGRIB, 
366                                ncep_gribfile=NCEP0P5_GRIB2_TESTFILE1)
367    if passed: passed_count += 1
368    else: failed_count +=1 
369    print("passed: %s" % passed)
370    print("-----------------------------------------------")
371
372
373    print("--------test_ncep_detects_missing_var_tsig1------")
374    passed = test_ncep_detects_missing_var_tsig1(
375                                checkgrib=CHECKGRIB, 
376                                ncep_gribfile=NCEP0P5_GRIB2_TESTFILE1)
377    if passed: passed_count += 1
378    else: failed_count +=1 
379    print("passed: %s" % passed)
380    print("-----------------------------------------------")
381
382
383
384    print("--------test_ncep_detects_grib1_2t------")
385    passed = test_ncep_detects_grib1_2t(
386                                checkgrib=CHECKGRIB, 
387                                ncep_gribfile=NCEP0P5_GRIB2_TESTFILE1)
388    if passed: passed_count += 1
389    else: failed_count +=1 
390    print("passed: %s" % passed)
391    print("-----------------------------------------------")
392
393
394
395
396    #----------------------
397    # NCEPFV3-specific tests
398    #----------------------
399
400    print("--------test_successful_ncepfv3grib2_check--------")
401    passed = test_successful_ncepfv3grib2_check(checkgrib=CHECKGRIB,
402                                gribfile=NCEPFV30P5_GRIB2_TESTFILE1)
403 
404    if passed:
405        passed_count += 1
406    else:
407        failed_count +=1
408        print('FV3 TEST FAILED - ABORT')
409        sys.exit()
410    print("passed: %s" % passed)
411    print("-----------------------------------------------")
412
413
414    print("--------test_detects_ncepfv3_more_pressure_levels_than_expected------")
415    passed = test_detects_ncepfv3_more_pressure_levels_than_expected(
416                                checkgrib=CHECKGRIB, 
417                                gribfile=NCEPFV30P5_GRIB2_TESTFILE1)
418    if passed: passed_count += 1
419    else: failed_count +=1 
420    print("passed: %s" % passed)
421    print("-----------------------------------------------")
422
423    print("--------test_detects_ncepfv3_fewer_pressure_levels_than_expected------")
424    passed = test_detects_ncepfv3_fewer_pressure_levels_than_expected(
425                                checkgrib=CHECKGRIB, 
426                                gribfile=NCEPFV30P5_GRIB2_TESTFILE1)
427    if passed: passed_count += 1
428    else: failed_count +=1 
429    print("passed: %s" % passed)
430    print("-----------------------------------------------")
431
432
433    print("--------test_ncepfv3_detects_missing_level_850------")
434    passed = test_ncepfv3_detects_missing_level_850(
435                                checkgrib=CHECKGRIB, 
436                                ncepfv3_gribfile=NCEPFV30P5_GRIB2_TESTFILE1)
437    if passed: passed_count += 1
438    else: failed_count +=1 
439    print("passed: %s" % passed)
440    print("-----------------------------------------------")
441
442
443    print("--------test_ncepfv3_detects_missing_level_100_r------")
444    passed = test_ncepfv3_detects_missing_level_100_r(
445                                checkgrib=CHECKGRIB, 
446                                ncepfv3_gribfile=NCEPFV30P5_GRIB2_TESTFILE1)
447    if passed: passed_count += 1
448    else: failed_count +=1 
449    print("passed: %s" % passed)
450    print("-----------------------------------------------")
451
452
453    print("--------test_ncepfv3_detects_missing_var_10u------")
454    passed = test_ncepfv3_detects_missing_var_10u(
455                                checkgrib=CHECKGRIB, 
456                                ncepfv3_gribfile=NCEPFV30P5_GRIB2_TESTFILE1)
457    if passed: passed_count += 1
458    else: failed_count +=1 
459    print("passed: %s" % passed)
460    print("-----------------------------------------------")
461
462
463    print("--------test_ncepfv3_detects_missing_var_tsig1------")
464    passed = test_ncepfv3_detects_missing_var_tsig1(
465                                checkgrib=CHECKGRIB, 
466                                ncepfv3_gribfile=NCEPFV30P5_GRIB2_TESTFILE1)
467    if passed: passed_count += 1
468    else: failed_count +=1 
469    print("passed: %s" % passed)
470    print("-----------------------------------------------")
471
472
473    print("--------test_ncepfv3_detects_grib1_2t------")
474    passed = test_ncepfv3_detects_grib1_2t(
475                                checkgrib=CHECKGRIB, 
476                                ncepfv3_gribfile=NCEPFV30P5_GRIB2_TESTFILE1)
477    if passed: passed_count += 1
478    else: failed_count +=1 
479    print("passed: %s" % passed)
480    print("-----------------------------------------------")
481
482
483
484
485    # Summary of tests passed/failed
486    print("\n************************")
487    print("Passed tests: %d" % passed_count)
488    print("Failed tests: %d" % failed_count)
489    print("************************\n")
490
491
492
493
494
495
496
497#############    General test functions    #####################
498
499
500def test_compile_checkgrib(checkgrib_dir=None):
501
502    """
503    Removes any existing copy of the binary, compiles, and tests
504    for presence, returning a True if successful, otherwise False
505
506    Expects executable to be named "gribcheck"
507
508    Assumes that Makefile in the compile directory is correct for the
509    environment
510    """
511
512    the_command = 'cd ' + checkgrib_dir + ' ; '
513    the_command += 'make clean > /dev/null 2>&1; '
514    the_command += 'make > /dev/null 2>&1; '
515    subprocess.call(the_command, shell=True)
516
517    execname = checkgrib_dir + '/checkGRIB'
518    if os.path.isfile(execname) and os.access(execname, os.X_OK):
519        success = True
520    else:
521        success = False
522
523    return success
524
525
526def test_missing_source_cli_arg_detected(checkgrib=None, gribfile=None):
527
528    """
529    checkgrib: full path to checkgrib executable
530    gribfile: full path to NCEP gribfile to check
531
532    Checks that utility exits with code 2 if the --source arg is missing
533    """
534
535    the_command = [checkgrib, 
536                              "--levels", "3",
537                              gribfile]
538
539    process = subprocess.Popen(the_command)
540    process.communicate()[0]
541    return_code = process.returncode
542
543    MY_LOGGER.info('return_code: ' + str(return_code))
544    if return_code == 2:
545        passed = True
546    else:
547        passed = False
548   
549    return passed
550
551
552def test_invalid_source_cli_arg_detected(checkgrib=None, gribfile=None):
553
554    """
555    checkgrib: full path to checkgrib executable
556    gribfile: full path to NCEP gribfile to check
557
558    Checks that utility exits with code 2 if the --source arg is invalid
559    """
560
561    the_command = [checkgrib, 
562                              "--levels", "3",
563                              "--source", "THIS_IS_INVALID!",
564                              gribfile]
565
566    process = subprocess.Popen(the_command)
567    process.communicate()[0]
568    return_code = process.returncode
569
570    MY_LOGGER.info('return_code: ' + str(return_code))
571    if return_code == 2:
572        passed = True
573    else:
574        passed = False
575   
576    return passed
577
578
579
580
581
582
583
584def test_missing_levels_cli_arg_detected(checkgrib=None, gribfile=None):
585
586    """
587    checkgrib: full path to checkgrib executable
588    gribfile: full path to NCEP gribfile to check
589
590    Checks that utility exits with code 2 if the --levels arg is missing
591    """
592
593    the_command = [checkgrib, 
594                              "--source", "ECMWF",
595                              gribfile]
596
597    process = subprocess.Popen(the_command)
598    process.communicate()[0]
599    return_code = process.returncode
600
601    MY_LOGGER.info('return_code: ' + str(return_code))
602    if return_code == 2:
603        passed = True
604    else:
605        passed = False
606   
607    return passed
608
609
610def test_checkanl_and_checkfcst_detected(checkgrib=None, gribfile=None):
611
612    """
613    checkgrib: full path to checkgrib executable
614    gribfile: full path to NCEP gribfile to check
615
616    Checks that utility exits with code 2 if both the --checkfcst and
617    --checkanl flags are specified
618    """
619
620    the_command = [checkgrib, 
621                              "--source", "ECMWF",
622                              "--levels", "137",
623                              "--checkanl", "--checkfcst",
624                              gribfile]
625
626    process = subprocess.Popen(the_command)
627    process.communicate()[0]
628    return_code = process.returncode
629
630    MY_LOGGER.info('return_code: ' + str(return_code))
631    if return_code == 2:
632        passed = True
633    else:
634        passed = False
635   
636    return passed
637
638
639
640
641def test_reports_no_cli_args(checkgrib=None):
642
643    """
644    checkgrib: full path to checkgrib executable
645
646    Checks that utility exits with code 2 if no CLI args were specified
647    """
648
649    the_command = [checkgrib] 
650
651    process = subprocess.Popen(the_command)
652    process.communicate()[0]
653    return_code = process.returncode
654
655    MY_LOGGER.info('return_code: ' + str(return_code))
656    if return_code == 2:
657        passed = True
658    else:
659        passed = False
660   
661    return passed
662
663
664
665def test_reports_no_path_args(checkgrib=None):
666
667    """
668    checkgrib: full path to checkgrib executable
669
670    Checks that utility exits with code 2 if no gribfile args were specified
671    """
672
673    the_command = [checkgrib, 
674                              "--source", "ECMWF",
675                              "--levels", "137"]
676
677    process = subprocess.Popen(the_command)
678    process.communicate()[0]
679    return_code = process.returncode
680
681    MY_LOGGER.info('return_code: ' + str(return_code))
682    if return_code == 2:
683        passed = True
684    else:
685        passed = False
686   
687    return passed
688
689
690
691
692
693
694
695
696
697
698
699
700
701#############    ECMWF test functions    #####################
702
703
704
705def test_reports_unexpected_ecmwf_grib_centre(checkgrib=None, 
706                                              ncep_gribfile=None):
707
708    """
709    Calls checkgrib for ECMWF source but file is actually NCEP
710
711    Expects an error message and return code of 1
712    """ 
713    the_command = [checkgrib, "--source", "ECMWF", 
714                              "--levels", "137",
715                              ncep_gribfile]
716
717    process = subprocess.Popen(the_command)
718    process.communicate()[0]
719    return_code = process.returncode
720
721    MY_LOGGER.info('return_code: ' + str(return_code))
722    if return_code == 1:
723        passed = True
724    else:
725        passed = False
726   
727    return passed
728
729
730def test_reports_missing_ecmwf_levels(checkgrib=None, 
731                                     ecmwf_gribfile=None):
732
733    """
734    Calls checkgrib for ECMWF specifying number of levels greater
735    than the expected 137
736
737    Expects an error message and return code of 1
738    """ 
739    the_command = [checkgrib, "--source", "ECMWF", 
740                              "--levels", "147",
741                              ecmwf_gribfile]
742
743    process = subprocess.Popen(the_command)
744    process.communicate()[0]
745    return_code = process.returncode
746
747    MY_LOGGER.info('return_code: ' + str(return_code))
748    if return_code == 1:
749        passed = True
750    else:
751        passed = False
752   
753    return passed
754
755
756
757
758
759
760
761def test_successful_ecmwfgrib2_check(checkgrib=None, gribfile=None):
762
763    """
764    checkgrib: full path to checkgrib executable
765    gribfile: full path to gribfile to check
766
767    Runs through a no-brainer successful test, expecting a 0 return code
768    """
769
770    the_command = [checkgrib, "--source", "ECMWF", 
771                              "--levels", "137",
772                              gribfile]
773
774    process = subprocess.Popen(the_command)
775    process.communicate()[0]
776    return_code = process.returncode
777
778    MY_LOGGER.info('return_code: ' + str(return_code))
779    if return_code == 0:
780        passed = True
781    else:
782        passed = False
783   
784    return passed
785
786
787def test_successful_ecmwfgrib_ef_check(checkgrib=None, ef_gribfile=None):
788
789    """
790    checkgrib: full path to checkgrib executable
791    ef_gribfile: full path to EF gribfile to check
792
793    Runs through a no-brainer successful test, expecting a 0 return code
794    The EF files contain both GRIB1 and GRIB2.  This tests that it will
795    still pass, as long as --checkgrib2 is not used
796    """
797
798    the_command = [checkgrib, "--source", "ECMWF", 
799                              "--levels", "137",
800                              ef_gribfile]
801
802    process = subprocess.Popen(the_command)
803    process.communicate()[0]
804    return_code = process.returncode
805
806    MY_LOGGER.info('return_code: ' + str(return_code))
807    if return_code == 0:
808        passed = True
809    else:
810        passed = False
811   
812    return passed
813
814
815
816def test_reports_ecmwfgrib_ef_fails_with_checkgrib2(checkgrib=None, ef_gribfile=None):
817
818    """
819    checkgrib: full path to checkgrib executable
820    ef_gribfile: full path to EF gribfile to check
821
822    Tests that EF check fails if checkgrib2 specified,
823    expecting a 1 return code
824    The EF files contain both GRIB1 and GRIB2.  This tests that it will
825    still pass, as long as --checkgrib2 is not used
826    """
827
828    the_command = [checkgrib, "--source", "ECMWF", 
829                              "--checkgrib2",
830                              "--levels", "137",
831                              ef_gribfile]
832
833    process = subprocess.Popen(the_command)
834    process.communicate()[0]
835    return_code = process.returncode
836
837    MY_LOGGER.info('return_code: ' + str(return_code))
838    if return_code == 1:
839        passed = True
840    else:
841        passed = False
842   
843    return passed
844
845
846
847
848
849def test_reports_unexpected_ecmwf_level(checkgrib=None, ecmwf_gribfile=None):
850
851    """
852    checkgrib: full path to checkgrib executable
853    gribfile: full path to NCEP gribfile to check
854
855    Checks an ECMWF file with "--levels 3", so exits when it encounters
856    a level higher than that.  Expected return_code is 1
857    """
858
859    the_command = [checkgrib, "--source", "ECMWF", 
860                              "--levels", "3",
861                              ecmwf_gribfile]
862
863    process = subprocess.Popen(the_command)
864    process.communicate()[0]
865    return_code = process.returncode
866
867    MY_LOGGER.info('return_code: ' + str(return_code))
868    if return_code == 1:
869        passed = True
870    else:
871        passed = False
872   
873    return passed
874
875
876def test_ecmwf_detects_missing_level_14(checkgrib=None, ecmwf_gribfile=None):
877
878    """
879    checkgrib: full path to checkgrib executable
880    gribfile: full path to NCEP gribfile to check
881
882    Checks an ECMWF file that has had vertical level 14 removed
883    Expected return_code is 1
884    """
885
886    # Create a temporary gribfile
887    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
888    the_command = 'grib_copy -w level!=14 '
889    the_command += str(ecmwf_gribfile) + ' ' + str(tempgribpath)
890    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
891    subprocess.call(the_command, shell=True)
892
893    # Use it
894    the_command = [checkgrib, "--source", "ECMWF", 
895                              "--levels", "137",
896                              tempgribpath]
897
898    process = subprocess.Popen(the_command)
899    process.communicate()[0]
900    return_code = process.returncode
901
902    MY_LOGGER.info('return_code: ' + str(return_code))
903    if return_code == 1:
904        passed = True
905    else:
906        passed = False
907
908    # Remove the temporary gribfile
909    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
910    os.remove(tempgribpath)
911   
912    return passed
913
914
915
916def test_ecmwf_detects_missing_var_lsp(checkgrib=None, ecmwf_gribfile=None):
917
918    """
919    checkgrib: full path to checkgrib executable
920    gribfile: full path to NCEP gribfile to check
921
922    Checks an ECMWF file that has had the lsp variable removed
923    Expected return_code is 1
924    """
925
926    # Create a temporary gribfile
927    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
928    the_command = 'grib_copy -w shortName!=lsp '
929    the_command += str(ecmwf_gribfile) + ' ' + str(tempgribpath)
930    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
931    subprocess.call(the_command, shell=True)
932
933    # Use it
934    the_command = [checkgrib, "--source", "ECMWF", 
935                              "--levels", "137",
936                              tempgribpath]
937
938    process = subprocess.Popen(the_command)
939    process.communicate()[0]
940    return_code = process.returncode
941
942    MY_LOGGER.info('return_code: ' + str(return_code))
943    if return_code == 1:
944        passed = True
945    else:
946        passed = False
947
948    # Remove the temporary gribfile
949    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
950    os.remove(tempgribpath)
951   
952    return passed
953
954
955
956
957
958
959
960
961
962def test_ecmwf_detects_missing_level_14_etadot(checkgrib=None, 
963                                               ecmwf_gribfile=None):
964
965    """
966    checkgrib: full path to checkgrib executable
967    gribfile: full path to NCEP gribfile to check
968
969    Checks an ECMWF file that has had vertical level 14 removed
970    just for etadot
971    Expected return_code is 1
972    """
973
974    # Create a temporary gribfile
975    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
976
977
978    the_command = "count=`grib_ls -p count -w level=14,shortName=etadot "
979    the_command += ecmwf_gribfile + " | sed -n 3p` ;"
980    the_command += " grib_copy -w count!=$count "
981    the_command += str(ecmwf_gribfile) + ' ' + str(tempgribpath)
982
983
984    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
985    MY_LOGGER.info('the_command: ' + str(the_command))
986    subprocess.call(the_command, shell=True)
987
988    # Use it
989    the_command = [checkgrib, "--source", "ECMWF", 
990                              "--levels", "137",
991                              tempgribpath]
992
993    process = subprocess.Popen(the_command)
994    process.communicate()[0]
995    return_code = process.returncode
996
997    MY_LOGGER.info('return_code: ' + str(return_code))
998    if return_code == 1:
999        passed = True
1000    else:
1001        passed = False
1002
1003    # Remove the temporary gribfile
1004    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
1005    os.remove(tempgribpath)
1006   
1007    return passed
1008
1009
1010
1011def test_ecmwf_detects_grib1_nsss(checkgrib=None, 
1012                                        ecmwf_gribfile=None):
1013
1014    """
1015    checkgrib: full path to checkgrib executable
1016    gribfile: full path to NCEP gribfile to check
1017
1018    Checks an ECMWF file that has message variable nsss
1019    changed from GRIB2 to GRIB1
1020    Expected return_code is 1
1021    """
1022
1023    # Create a temporary gribfile
1024    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
1025
1026    the_command = "grib_set -s edition=1 -w shortName=nsss "
1027    the_command += str(ecmwf_gribfile) + ' ' + str(tempgribpath)
1028
1029
1030    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
1031    MY_LOGGER.info('the_command: ' + str(the_command))
1032    subprocess.call(the_command, shell=True)
1033
1034    # Use it
1035    the_command = [checkgrib, "--source", "ECMWF", 
1036                              "--checkgrib2",
1037                              "--levels", "137",
1038                              tempgribpath]
1039
1040    process = subprocess.Popen(the_command)
1041    process.communicate()[0]
1042    return_code = process.returncode
1043
1044    MY_LOGGER.info('return_code: ' + str(return_code))
1045    if return_code == 1:
1046        passed = True
1047    else:
1048        passed = False
1049
1050    # Remove the temporary gribfile
1051    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
1052    os.remove(tempgribpath)
1053   
1054    return passed
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073def test_reports_failed_ecmwf_anl_check(checkgrib=None, 
1074                                        ecmwf_fcst_gribfile=None):
1075
1076    """
1077    checkgrib: full path to checkgrib executable
1078    gribfile: full path to ECMWF fcst gribfile to check
1079
1080    Checks a fcst ECMWF file with "--checkanl", so exits when it encounters
1081    fcst.  Expected return_code is 1
1082    """
1083
1084    the_command = [checkgrib, "--source", "ECMWF", 
1085                              "--levels", "137",
1086                              "--checkanl",
1087                              ecmwf_fcst_gribfile]
1088
1089    process = subprocess.Popen(the_command)
1090    process.communicate()[0]
1091    return_code = process.returncode
1092
1093    MY_LOGGER.info('return_code: ' + str(return_code))
1094    if return_code == 1:
1095        passed = True
1096    else:
1097        passed = False
1098   
1099    return passed
1100
1101
1102
1103def test_reports_failed_ecmwf_fcst_check(checkgrib=None, 
1104                                        ecmwf_anl_gribfile=None):
1105
1106    """
1107    checkgrib: full path to checkgrib executable
1108    gribfile: full path to ECMWF anl gribfile to check
1109
1110    Checks a anl ECMWF file with "--checkfcst", so exits when it encounters
1111    anl.  Expected return_code is 1
1112    """
1113
1114    the_command = [checkgrib, "--source", "ECMWF", 
1115                              "--levels", "137",
1116                              "--checkfcst",
1117                              ecmwf_anl_gribfile]
1118
1119    process = subprocess.Popen(the_command)
1120    process.communicate()[0]
1121    return_code = process.returncode
1122
1123    MY_LOGGER.info('return_code: ' + str(return_code))
1124    if return_code == 1:
1125        passed = True
1126    else:
1127        passed = False
1128   
1129    return passed
1130
1131
1132
1133def test_ecmwf_successful_fcst_check(checkgrib=None, ecmwf_fcst_gribfile=None):
1134
1135    """
1136    checkgrib: full path to checkgrib executable
1137    ecmwf_fcst_gribfile: full path to ECMWF fcst gribfile to check
1138
1139    Checks a fcst ECMWF file with "--checkfcst", so exits normally.
1140    Expected return_code is 0
1141    """
1142
1143    ERROR_MSG = """
1144    ****************************************************************
1145    WARNING: With current CTBTO ECMWF files, this test will always
1146             fail, because the files have mixed anl and fcst messages.
1147
1148             YOU SHOULD NOT USE THIS TEST UNTIL THE SITUATION IS RESOLVED
1149
1150    BuhBye
1151    """
1152
1153    print(ERROR_MSG)
1154    sys.exit()
1155
1156
1157
1158    the_command = [checkgrib, "--source", "ECMWF", 
1159                              "--levels", "137",
1160                              "--checkfcst",
1161                              ecmwf_fcst_gribfile]
1162
1163    process = subprocess.Popen(the_command)
1164    process.communicate()[0]
1165    return_code = process.returncode
1166
1167    MY_LOGGER.info('return_code: ' + str(return_code))
1168    if return_code == 0:
1169        passed = True
1170    else:
1171        passed = False
1172   
1173    return passed
1174
1175
1176
1177
1178def test_ecmwf_successful_anl_check(checkgrib=None, ecmwf_anl_gribfile=None):
1179
1180    """
1181    checkgrib: full path to checkgrib executable
1182    ecmwf_anl_gribfile: full path to ECMWF anl gribfile to check
1183
1184    Checks a anl ECMWF file with "--checkanl", so exits normally.
1185    Expected return_code is 0
1186    """
1187
1188    ERROR_MSG = """
1189    ****************************************************************
1190    WARNING: With current CTBTO ECMWF files, this test will always
1191             fail, because the files have mixed anl and fcst messages.
1192
1193             YOU SHOULD NOT USE THIS TEST UNTIL THE SITUATION IS RESOLVED
1194
1195    BuhBye
1196    """
1197
1198    print(ERROR_MSG)
1199    sys.exit()
1200
1201
1202
1203    the_command = [checkgrib, "--source", "ECMWF", 
1204                              "--levels", "137",
1205                              "--checkanl",
1206                              ecmwf_anl_gribfile]
1207
1208    process = subprocess.Popen(the_command)
1209    process.communicate()[0]
1210    return_code = process.returncode
1211
1212    MY_LOGGER.info('return_code: ' + str(return_code))
1213    if return_code == 0:
1214        passed = True
1215    else:
1216        passed = False
1217   
1218    return passed
1219
1220
1221
1222
1223
1224#############    NCEP test functions    #####################
1225
1226def test_reports_unexpected_ncep_grib_centre(checkgrib=None, 
1227                                             ecmwf_gribfile=None):
1228
1229    """
1230    Calls checkgrib for NCEP source but file is actually ECMWF
1231
1232    Expects an error message and return code of 1
1233    """ 
1234    the_command = [checkgrib, "--source", "NCEP", 
1235                              "--levels", "30",
1236                              ecmwf_gribfile]
1237
1238    process = subprocess.Popen(the_command)
1239    process.communicate()[0]
1240    return_code = process.returncode
1241
1242    MY_LOGGER.info('return_code: ' + str(return_code))
1243    if return_code == 1:
1244        passed = True
1245    else:
1246        passed = False
1247   
1248    return passed
1249
1250
1251def test_successful_ncepgrib2_check(checkgrib=None, 
1252                                    gribfile=None):
1253
1254    """
1255    checkgrib: full path to checkgrib executable
1256    gribfile: full path to gribfile to check
1257
1258    Runs through a no-brainer successful test, expecting a 0 return code
1259    """
1260
1261    the_command = [checkgrib, "--source", "NCEP", 
1262                              "--checkgrib2",
1263                              "--levels", "31",
1264                              gribfile]
1265
1266    process = subprocess.Popen(the_command)
1267    process.communicate()[0]
1268    return_code = process.returncode
1269
1270    MY_LOGGER.info('return_code: ' + str(return_code))
1271    if return_code == 0:
1272        passed = True
1273    else:
1274        passed = False
1275   
1276    return passed
1277
1278
1279
1280def test_reports_ncep_bad_level_structure(checkgrib=None, 
1281                                          ncep_gribfile=None):
1282
1283    """
1284    Tests that if the pressure level structure of the NCEP file does
1285    not agree with expected number of levels, an error will be reported
1286
1287    Expects a return code of 1
1288    """ 
1289
1290    print("Not yet implemented...")
1291
1292
1293
1294
1295def test_detects_ncep_more_pressure_levels_than_expected(
1296                                    checkgrib=None, 
1297                                    gribfile=None):
1298
1299    """
1300    checkgrib: full path to checkgrib executable
1301    gribfile: full path to gribfile to check
1302
1303    Tests that a return code of 1 will result if checkgrib finds
1304    more pressure levels than specified in command line
1305    """
1306
1307    the_command = [checkgrib, "--source", "NCEP", 
1308                              "--levels", "30",
1309                              gribfile]
1310
1311    process = subprocess.Popen(the_command)
1312    process.communicate()[0]
1313    return_code = process.returncode
1314
1315    MY_LOGGER.info('return_code: ' + str(return_code))
1316    if return_code == 1:
1317        passed = True
1318    else:
1319        passed = False
1320   
1321    return passed
1322
1323
1324
1325def test_detects_ncep_fewer_pressure_levels_than_expected(
1326                                    checkgrib=None, 
1327                                    gribfile=None):
1328
1329    """
1330    checkgrib: full path to checkgrib executable
1331    gribfile: full path to gribfile to check
1332
1333    Tests that a return code of 1 will result if checkgrib finds
1334    fewer pressure levels than specified in command line
1335    """
1336
1337    the_command = [checkgrib, "--source", "NCEP", 
1338                              "--levels", "32",
1339                              gribfile]
1340
1341    process = subprocess.Popen(the_command)
1342    process.communicate()[0]
1343    return_code = process.returncode
1344
1345    MY_LOGGER.info('return_code: ' + str(return_code))
1346    if return_code == 1:
1347        passed = True
1348    else:
1349        passed = False
1350   
1351    return passed
1352
1353
1354
1355def test_ncep_detects_missing_level_850(checkgrib=None, ncep_gribfile=None):
1356
1357    """
1358    checkgrib: full path to checkgrib executable
1359    ncep_gribfile: full path to NCEP gribfile to check
1360
1361    Checks an NCEP file that has had vertical level 850 removed
1362    Expected return_code is 1
1363
1364    Note, as currently implemented, checkgrib will not see any 850 mb
1365    levels, and so it won't add them into expected inventory.  It will
1366    fail because it won't see the 31 levels it expects.
1367    """
1368
1369    # Create a temporary gribfile
1370    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
1371    the_command = 'grib_copy -w level!=850 '
1372    the_command += str(ncep_gribfile) + ' ' + str(tempgribpath)
1373    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
1374    subprocess.call(the_command, shell=True)
1375
1376    # Use it
1377    the_command = [checkgrib, "--source", "NCEP", 
1378                              "--levels", "31",
1379                              tempgribpath]
1380
1381    process = subprocess.Popen(the_command)
1382    process.communicate()[0]
1383    return_code = process.returncode
1384
1385    MY_LOGGER.info('return_code: ' + str(return_code))
1386    if return_code == 1:
1387        passed = True
1388    else:
1389        passed = False
1390
1391    # Remove the temporary gribfile
1392    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
1393    os.remove(tempgribpath)
1394   
1395    return passed
1396
1397
1398def test_ncep_detects_missing_level_100_r(checkgrib=None, ncep_gribfile=None):
1399
1400    """
1401    checkgrib: full path to checkgrib executable
1402    ncep_gribfile: full path to NCEP gribfile to check
1403
1404    Checks an NCEP file that has had vertical level 100 removed
1405    for variable r
1406    Expected return_code is 1
1407    """
1408
1409    # Create a temporary gribfile
1410    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
1411
1412    the_command = "count=`grib_ls -p count -w level=100,shortName=r "
1413    the_command += ncep_gribfile + " | sed -n 3p` ;"
1414    the_command += " grib_copy -w count!=$count "
1415    the_command += str(ncep_gribfile) + ' ' + str(tempgribpath)
1416
1417    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
1418    subprocess.call(the_command, shell=True)
1419
1420    # Use it
1421    the_command = [checkgrib, "--source", "NCEP", 
1422                              "--levels", "31",
1423                              tempgribpath]
1424
1425    process = subprocess.Popen(the_command)
1426    process.communicate()[0]
1427    return_code = process.returncode
1428
1429    MY_LOGGER.info('return_code: ' + str(return_code))
1430    if return_code == 1:
1431        passed = True
1432    else:
1433        passed = False
1434
1435    # Remove the temporary gribfile
1436    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
1437    os.remove(tempgribpath)
1438   
1439    return passed
1440
1441
1442
1443def test_ncep_detects_missing_var_10u(checkgrib=None, ncep_gribfile=None):
1444
1445    """
1446    checkgrib: full path to checkgrib executable
1447    ncep_gribfile: full path to NCEP gribfile to check
1448
1449    Checks an NCEP file that has had 10m U removed
1450    Expected return_code is 1
1451    """
1452
1453    # Create a temporary gribfile
1454    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
1455
1456    the_command = 'grib_copy -w shortName!=10u '
1457    the_command += str(ncep_gribfile) + ' ' + str(tempgribpath)
1458
1459    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
1460    subprocess.call(the_command, shell=True)
1461
1462    # Use it
1463    the_command = [checkgrib, "--source", "NCEP", 
1464                              "--levels", "31",
1465                              tempgribpath]
1466
1467    process = subprocess.Popen(the_command)
1468    process.communicate()[0]
1469    return_code = process.returncode
1470
1471    MY_LOGGER.info('return_code: ' + str(return_code))
1472    if return_code == 1:
1473        passed = True
1474    else:
1475        passed = False
1476
1477    # Remove the temporary gribfile
1478    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
1479    os.remove(tempgribpath)
1480   
1481    return passed
1482
1483
1484def test_ncep_detects_missing_var_tsig1(checkgrib=None, ncep_gribfile=None):
1485
1486    """
1487    checkgrib: full path to checkgrib executable
1488    ncep_gribfile: full path to NCEP gribfile to check
1489
1490    Checks an NCEP file that has had sigma level t removed
1491    Expected return_code is 1
1492    """
1493
1494    # Create a temporary gribfile
1495    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
1496
1497
1498    the_command = "count=`grib_ls -p count -w typeOfLevel=sigma,shortName=t "
1499    the_command += ncep_gribfile + " | sed -n 3p` ;"
1500    the_command += " grib_copy -w count!=$count "
1501    the_command += str(ncep_gribfile) + ' ' + str(tempgribpath)
1502
1503
1504    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
1505    subprocess.call(the_command, shell=True)
1506
1507    # Use it
1508    the_command = [checkgrib, "--source", "NCEP", 
1509                              "--levels", "31",
1510                              tempgribpath]
1511
1512    process = subprocess.Popen(the_command)
1513    process.communicate()[0]
1514    return_code = process.returncode
1515
1516    MY_LOGGER.info('return_code: ' + str(return_code))
1517    if return_code == 1:
1518        passed = True
1519    else:
1520        passed = False
1521
1522    # Remove the temporary gribfile
1523    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
1524    os.remove(tempgribpath)
1525   
1526    return passed
1527
1528
1529
1530def test_ncep_detects_grib1_2t(checkgrib=None, ncep_gribfile=None):
1531
1532    """
1533    checkgrib: full path to checkgrib executable
1534    ncep_gribfile: full path to NCEP gribfile to check
1535
1536    Checks an NCEP file that has had the 2t (2m T) message changed from
1537    GRIB2 to GRIB1
1538    Expected return_code is 1
1539    """
1540
1541    # Create a temporary gribfile
1542    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
1543
1544    the_command = "grib_set -s edition=1,packingType=grid_simple "
1545    the_command += " -w shortName=2t "
1546    the_command += str(ncep_gribfile) + ' ' + str(tempgribpath)
1547
1548
1549    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
1550    subprocess.call(the_command, shell=True)
1551
1552    # Use it
1553    the_command = [checkgrib, "--source", "NCEP", 
1554                              "--checkgrib2",
1555                              "--levels", "31",
1556                              tempgribpath]
1557
1558    process = subprocess.Popen(the_command)
1559    process.communicate()[0]
1560    return_code = process.returncode
1561
1562    MY_LOGGER.info('return_code: ' + str(return_code))
1563    if return_code == 1:
1564        passed = True
1565    else:
1566        passed = False
1567
1568    # Remove the temporary gribfile
1569    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
1570    os.remove(tempgribpath)
1571   
1572    return passed
1573
1574
1575
1576
1577#############    NCEPFV3 test functions    #####################
1578
1579def test_successful_ncepfv3grib2_check(checkgrib=None,
1580                                    gribfile=None):
1581
1582    """
1583    checkgrib: full path to checkgrib executable
1584    gribfile: full path to gribfile to check
1585
1586    Runs through a no-brainer successful test, expecting a 0 return code
1587    """
1588
1589    the_command = [checkgrib, "--source", "NCEPFV3",
1590                              "--checkgrib2",
1591                              "--levels", "31",
1592                              gribfile]
1593
1594    process = subprocess.Popen(the_command)
1595    process.communicate()[0]
1596    return_code = process.returncode
1597
1598    MY_LOGGER.info('return_code: ' + str(return_code))
1599    if return_code == 0:
1600        passed = True
1601    else:
1602        passed = False
1603
1604    return passed
1605
1606
1607def test_detects_ncepfv3_more_pressure_levels_than_expected(
1608                                    checkgrib=None, 
1609                                    gribfile=None):
1610
1611    """
1612    checkgrib: full path to checkgrib executable
1613    gribfile: full path to gribfile to check
1614
1615    Tests that a return code of 1 will result if checkgrib finds
1616    more pressure levels than specified in command line
1617    """
1618
1619    the_command = [checkgrib, "--source", "NCEPFV3", 
1620                              "--levels", "30",
1621                              gribfile]
1622
1623    process = subprocess.Popen(the_command)
1624    process.communicate()[0]
1625    return_code = process.returncode
1626
1627    MY_LOGGER.info('return_code: ' + str(return_code))
1628    if return_code == 1:
1629        passed = True
1630    else:
1631        passed = False
1632   
1633    return passed
1634
1635
1636
1637def test_detects_ncepfv3_fewer_pressure_levels_than_expected(
1638                                    checkgrib=None, 
1639                                    gribfile=None):
1640
1641    """
1642    checkgrib: full path to checkgrib executable
1643    gribfile: full path to gribfile to check
1644
1645    Tests that a return code of 1 will result if checkgrib finds
1646    fewer pressure levels than specified in command line
1647    """
1648
1649    the_command = [checkgrib, "--source", "NCEPFV3", 
1650                              "--levels", "32",
1651                              gribfile]
1652
1653    process = subprocess.Popen(the_command)
1654    process.communicate()[0]
1655    return_code = process.returncode
1656
1657    MY_LOGGER.info('return_code: ' + str(return_code))
1658    if return_code == 1:
1659        passed = True
1660    else:
1661        passed = False
1662   
1663    return passed
1664
1665def test_ncepfv3_detects_missing_level_850(checkgrib=None, ncepfv3_gribfile=None):
1666
1667    """
1668    checkgrib: full path to checkgrib executable
1669    ncepfv3_gribfile: full path to NCEP FV3 gribfile to check
1670
1671    Checks an NCEP FV3 file that has had vertical level 850 removed
1672    Expected return_code is 1
1673
1674    Note, as currently implemented, checkgrib will not see any 850 mb
1675    levels, and so it won't add them into expected inventory.  It will
1676    fail because it won't see the 31 levels it expects.
1677    """
1678
1679    # Create a temporary gribfile
1680    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
1681    the_command = 'grib_copy -w level!=850 '
1682    the_command += str(ncepfv3_gribfile) + ' ' + str(tempgribpath)
1683    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
1684    subprocess.call(the_command, shell=True)
1685
1686    # Use it
1687    the_command = [checkgrib, "--source", "NCEPFV3", 
1688                              "--levels", "31",
1689                              tempgribpath]
1690
1691    process = subprocess.Popen(the_command)
1692    process.communicate()[0]
1693    return_code = process.returncode
1694
1695    MY_LOGGER.info('return_code: ' + str(return_code))
1696    if return_code == 1:
1697        passed = True
1698    else:
1699        passed = False
1700
1701    # Remove the temporary gribfile
1702    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
1703    os.remove(tempgribpath)
1704   
1705    return passed
1706
1707
1708def test_ncepfv3_detects_missing_level_100_r(checkgrib=None, ncepfv3_gribfile=None):
1709
1710    """
1711    checkgrib: full path to checkgrib executable
1712    ncepfv3_gribfile: full path to NCEP FV3 gribfile to check
1713
1714    Checks an NCEP FV3 file that has had vertical level 100 removed
1715    for variable r
1716    Expected return_code is 1
1717    """
1718
1719    # Create a temporary gribfile
1720    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
1721
1722    the_command = "count=`grib_ls -p count -w level=100,shortName=r "
1723    the_command += ncepfv3_gribfile + " | sed -n 3p` ;"
1724    the_command += " grib_copy -w count!=$count "
1725    the_command += str(ncepfv3_gribfile) + ' ' + str(tempgribpath)
1726
1727    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
1728    subprocess.call(the_command, shell=True)
1729
1730    # Use it
1731    the_command = [checkgrib, "--source", "NCEPFV3", 
1732                              "--levels", "31",
1733                              tempgribpath]
1734
1735    process = subprocess.Popen(the_command)
1736    process.communicate()[0]
1737    return_code = process.returncode
1738
1739    MY_LOGGER.info('return_code: ' + str(return_code))
1740    if return_code == 1:
1741        passed = True
1742    else:
1743        passed = False
1744
1745    # Remove the temporary gribfile
1746    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
1747    os.remove(tempgribpath)
1748   
1749    return passed
1750
1751
1752def test_ncepfv3_detects_missing_var_10u(checkgrib=None, ncepfv3_gribfile=None):
1753
1754    """
1755    checkgrib: full path to checkgrib executable
1756    ncepfv3_gribfile: full path to NCEP FV3 gribfile to check
1757
1758    Checks an NCEP FV3 file that has had 10m U removed
1759    Expected return_code is 1
1760    """
1761
1762    # Create a temporary gribfile
1763    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
1764
1765    the_command = 'grib_copy -w shortName!=10u '
1766    the_command += str(ncepfv3_gribfile) + ' ' + str(tempgribpath)
1767
1768    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
1769    subprocess.call(the_command, shell=True)
1770
1771    # Use it
1772    the_command = [checkgrib, "--source", "NCEPFV3", 
1773                              "--levels", "31",
1774                              tempgribpath]
1775
1776    process = subprocess.Popen(the_command)
1777    process.communicate()[0]
1778    return_code = process.returncode
1779
1780    MY_LOGGER.info('return_code: ' + str(return_code))
1781    if return_code == 1:
1782        passed = True
1783    else:
1784        passed = False
1785
1786    # Remove the temporary gribfile
1787    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
1788    os.remove(tempgribpath)
1789   
1790    return passed
1791
1792
1793def test_ncepfv3_detects_missing_var_tsig1(checkgrib=None, ncepfv3_gribfile=None):
1794
1795    """
1796    checkgrib: full path to checkgrib executable
1797    ncepfv3_gribfile: full path to NCEP FV3 gribfile to check
1798
1799    Checks an NCEP FV3 file that has had sigma level t removed
1800    Expected return_code is 1
1801    """
1802
1803    # Create a temporary gribfile
1804    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
1805
1806
1807    the_command = "count=`grib_ls -p count -w typeOfLevel=sigma,shortName=t "
1808    the_command += ncepfv3_gribfile + " | sed -n 3p` ;"
1809    the_command += " grib_copy -w count!=$count "
1810    the_command += str(ncepfv3_gribfile) + ' ' + str(tempgribpath)
1811
1812
1813    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
1814    subprocess.call(the_command, shell=True)
1815
1816    # Use it
1817    the_command = [checkgrib, "--source", "NCEPFV3", 
1818                              "--levels", "31",
1819                              tempgribpath]
1820
1821    process = subprocess.Popen(the_command)
1822    process.communicate()[0]
1823    return_code = process.returncode
1824
1825    MY_LOGGER.info('return_code: ' + str(return_code))
1826    if return_code == 1:
1827        passed = True
1828    else:
1829        passed = False
1830
1831    # Remove the temporary gribfile
1832    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
1833    os.remove(tempgribpath)
1834   
1835    return passed
1836
1837
1838def test_ncepfv3_detects_grib1_2t(checkgrib=None, ncepfv3_gribfile=None):
1839
1840    """
1841    checkgrib: full path to checkgrib executable
1842    ncepfv3_gribfile: full path to NCEP FV3 gribfile to check
1843
1844    Checks an NCEP FV3 file that has had the 2t (2m T) message changed from
1845    GRIB2 to GRIB1
1846    Expected return_code is 1
1847    """
1848
1849    # Create a temporary gribfile
1850    tempgribpath = '/tmp/' + str(uuid.uuid4()) + '.gr2'
1851
1852    the_command = "grib_set -s edition=1,packingType=grid_simple "
1853    the_command += " -w shortName=2t "
1854    the_command += str(ncepfv3_gribfile) + ' ' + str(tempgribpath)
1855
1856
1857    MY_LOGGER.info('Creating tempgribpath: ' + str(tempgribpath))
1858    subprocess.call(the_command, shell=True)
1859
1860    # Use it
1861    the_command = [checkgrib, "--source", "NCEPFV3", 
1862                              "--checkgrib2",
1863                              "--levels", "31",
1864                              tempgribpath]
1865
1866    process = subprocess.Popen(the_command)
1867    process.communicate()[0]
1868    return_code = process.returncode
1869
1870    MY_LOGGER.info('return_code: ' + str(return_code))
1871    if return_code == 1:
1872        passed = True
1873    else:
1874        passed = False
1875
1876    # Remove the temporary gribfile
1877    MY_LOGGER.info('Removing tempgribpath: ' + str(tempgribpath))
1878    os.remove(tempgribpath)
1879   
1880    return passed
1881
1882
1883
1884
1885
1886if __name__=="__main__":
1887
1888    main()
1889
1890
Note: See TracBrowser for help on using the repository browser.
hosted by ZAMG