Changes in Source/Python/Mods/tools.py [026b359:75db9b0] in flex_extract.git
- File:
-
- 1 edited
-
Source/Python/Mods/tools.py (modified) (30 diffs)
Legend:
- Unmodified
- Added
- Removed
-
Source/Python/Mods/tools.py
r026b359 r75db9b0 18 18 # - added documentation 19 19 # - moved all non class methods from former file Flexparttools in here 20 # - sep erated args and control interpretation20 # - separated args and control interpretation 21 21 # - added functions get_list_as_string, read_ecenv, send_mail, make_dir, 22 22 # put_file_to_ecserver, submit_job_to_ecserver, get_informations, 23 23 # get_dimensions, execute_subprocess, none_or_int, none_or_str 24 # 25 # August 2020 - Leopold Haimberger (University of Vienna) 26 # - added function to check if a specific string is in a file 27 # - added function to overwrite lines in a file which contain specific string 24 28 # 25 29 # @License: … … 118 122 119 123 def none_or_str(value): 120 '''Converts the input string into pythons None-type if the string121 contains string "None".124 '''Converts the input string into Pythons None type if it 125 contains the string "None". 122 126 123 127 Parameters … … 130 134 None or value: 131 135 Return depends on the content of the input value. If it was "None", 132 then the python type None is returned. Otherwise the string itself.136 then the Python type None is returned, otherwise the string itself. 133 137 ''' 134 138 if value == 'None': … … 137 141 138 142 def none_or_int(value): 139 '''Converts the input string into pythons None-type if the string140 contains string "None" . Otherwise it is converted to an integer value.143 '''Converts the input string into Pythons None-type if it 144 contains string "None"; otherwise it is converted to an integer value. 141 145 142 146 Parameters … … 158 162 def get_cmdline_args(): 159 163 '''Decomposes the command line arguments and assigns them to variables. 160 Apply default values for non mentioned arguments.164 Apply default values for arguments not present. 161 165 162 166 Parameters … … 166 170 ------ 167 171 args : Namespace 168 Contains the command line arguments from script/program call.172 Contains the command line arguments from the script / program call. 169 173 ''' 170 174 … … 191 195 parser.add_argument("--basetime", dest="basetime", 192 196 type=none_or_int, default=None, 193 help="base such as 0 or 12 (for half day retrievals)")197 help="base time such as 0 or 12 (for half day retrievals)") 194 198 parser.add_argument("--step", dest="step", 195 199 type=none_or_str, default=None, … … 200 204 parser.add_argument("--area", dest="area", 201 205 type=none_or_str, default=None, 202 help="area defined asnorth/west/south/east")206 help="area, defined by north/west/south/east") 203 207 204 208 # some switches 205 209 parser.add_argument("--debug", dest="debug", 206 210 type=none_or_int, default=None, 207 help="debug mode - leave temporary files intact")211 help="debug mode - temporary files will be conserved") 208 212 parser.add_argument("--oper", dest="oper", 209 213 type=none_or_int, default=None, 210 help='operational mode - prepares dates with'214 help='operational mode - prepares dates from ' 211 215 'environment variables') 212 216 parser.add_argument("--request", dest="request", 213 217 type=none_or_int, default=None, 214 help="list all marsrequests in file mars_requests.dat")218 help="list all MARS requests in file mars_requests.dat") 215 219 parser.add_argument("--public", dest="public", 216 220 type=none_or_int, default=None, 217 help="public mode - retrieves thepublic datasets")221 help="public mode - retrieves public datasets") 218 222 parser.add_argument("--rrint", dest="rrint", 219 223 type=none_or_int, default=None, 220 help='Selection of old or new precipitation'221 'interpolation :\n'224 help='Selection of old or new ' 225 'interpolation method for precipitation:\n' 222 226 ' 0 - old method\n' 223 227 ' 1 - new method (additional subgrid points)') … … 226 230 parser.add_argument("--inputdir", dest="inputdir", 227 231 type=none_or_str, default=None, 228 help='Path to t he temporary directory for the'229 'retriev algrib files and other processing files.')232 help='Path to temporary directory for ' 233 'retrieved grib files and other processing files.') 230 234 parser.add_argument("--outputdir", dest="outputdir", 231 235 type=none_or_str, default=None, 232 help='Path to the final directory where the final'233 'FLEXPART ready input files are stored.')236 help='Path to final directory where ' 237 'FLEXPART input files will be stored.') 234 238 235 239 # this is only used by prepare_flexpart.py to rerun a postprocessing step 236 240 parser.add_argument("--ppid", dest="ppid", 237 241 type=none_or_str, default=None, 238 help='This is the specify parent process id of a '242 help='This is the specify the parent process id of a ' 239 243 'single flex_extract run to identify the files. ' 240 244 'It is the second number in the GRIB files.') … … 243 247 parser.add_argument("--job_template", dest='job_template', 244 248 type=none_or_str, default="job.temp", 245 help='The job template file which are adapted to be ' 246 'submitted to the batch system on ECMWF server.') 249 help='Job template file. Will be used for submission ' 250 'to the batch system on the ECMWF server after ' 251 'modification.') 247 252 parser.add_argument("--queue", dest="queue", 248 253 type=none_or_str, default=None, 249 help='The ECMWF server name for submission of the'250 'job script to the batch system'254 help='The name of the ECMWF server name where the' 255 'job script is to be submitted ' 251 256 '(e.g. ecgate | cca | ccb)') 252 257 … … 289 294 290 295 It keeps the final FLEXPART input files if program runs without 291 ECMWF A piand keywords "ectrans" or "ecstorage" are set to "1".296 ECMWF API and keywords "ectrans" or "ecstorage" are set to "1". 292 297 293 298 Parameters … … 410 415 411 416 This method combines the single characters of the passed arguments 412 with each other . Sothat each character of each argument value417 with each other in a way that each character of each argument value 413 418 will be combined with each character of the other arguments as a tuple. 414 419 … … 519 524 pars : str 520 525 Addpar argument from CONTROL file in the format of 521 parameter names instead of ids. The parameter short 522 names are sepearted with "/" and they are passed as 523 one single string. 526 parameter names instead of IDs. The parameter short 527 names are separated by "/" and passed as one single string. 524 528 525 529 table : dict … … 555 559 556 560 def to_param_id_with_tablenumber(pars, table): 557 '''Transform parameter names to parameter ids and add table id.561 '''Transform parameter names to parameter IDs and add table ID. 558 562 559 563 Conversion with ECMWF grib table 128. … … 563 567 pars : str 564 568 Addpar argument from CONTROL file in the format of 565 parameter names instead of ids. The parameter short 566 names are sepearted with "/" and they are passed as 567 one single string. 569 parameter names instead of ID. The parameter short 570 names are separated by "/" and passed as one single string. 568 571 569 572 table : dict … … 576 579 spar : str 577 580 List of addpar parameters from CONTROL file transformed to 578 parameter ids in the format of integer.581 parameter IDs in the format of integer. 579 582 ''' 580 583 if not pars: … … 598 601 599 602 def get_list_as_string(list_obj, concatenate_sign=', '): 600 '''Converts a list of arbitrary content into a single string. 603 '''Converts a list of arbitrary content into a single string using a given 604 concatenation character. 601 605 602 606 Parameters … … 624 628 '''Creates a directory. 625 629 626 I t gives a warning if the directory already exists and skips process.627 The program stops only if there is another problem.630 If the directory already exists, an information is printed and the creation 631 skipped. The program stops only if there is another problem. 628 632 629 633 Parameters … … 641 645 # errno.EEXIST = directory already exists 642 646 if e.errno == errno.EEXIST: 643 print(' WARNING: Directory {0} already exists!'.format(directory))647 print('INFORMATION: Directory {0} already exists!'.format(directory)) 644 648 else: 645 649 raise # re-raise exception if a different error occured … … 720 724 ------ 721 725 job_id : int 722 The id number of the job as a reference at the ecmwfserver.726 The id number of the job as a reference at the ECMWF server. 723 727 ''' 724 728 … … 732 736 733 737 print('\n... Do you have a valid ecaccess certification key?') 734 sys.exit('... ECACCESS-JOB-SUBMITFAILED!')738 sys.exit('... ecaccess-job-submit FAILED!') 735 739 except OSError as e: 736 740 print('... ERROR CODE: ' + str(e.errno)) … … 738 742 739 743 print('\n... Most likely the ECACCESS library is not available!') 740 sys.exit('... ECACCESS-JOB-SUBMITFAILED!')744 sys.exit('... ecaccess-job-submit FAILED!') 741 745 742 746 return job_id.decode() … … 744 748 745 749 def get_informations(filename): 746 ''' Gets basic information from an example grib file.747 748 Th ese information are importantfor later use and the749 initialization of numpy arrays for data storing.750 '''Extracts basic information from a sample grib file. 751 752 This information is needed for later use and the 753 initialization of numpy arrays where data are stored. 750 754 751 755 Parameters … … 768 772 769 773 # --- open file --- 770 print("Opening file for getting information data--- %s" % filename)774 print("Opening grib file for extraction of information --- %s" % filename) 771 775 with open(filename, 'rb') as f: 772 776 # load first message from file … … 785 789 ] 786 790 787 print('\nInformation s are: ')791 print('\nInformation extracted: ') 788 792 for key in keys: 789 793 # Get the value of the key in a grib message. … … 798 802 799 803 def get_dimensions(info, purefc, dtime, index_vals, start_date, end_date): 800 '''This function specifies the correct dimensions for x, y and t.804 '''This function specifies the correct dimensions for x, y, and t. 801 805 802 806 Parameters … … 817 821 index_vals : list of list of str 818 822 Contains the values from the keys used for a distinct selection 819 of grib messages in processingthe grib files.823 of GRIB messages in processing the grib files. 820 824 Content looks like e.g.: 821 825 index_vals[0]: ('20171106', '20171107', '20171108') ; date … … 849 853 850 854 def execute_subprocess(cmd_list, error_msg='SUBPROCESS FAILED!'): 851 '''Executes a command line instructionvia a subprocess.855 '''Executes a command via a subprocess. 852 856 853 857 Error handling is done if an error occures. … … 856 860 ---------- 857 861 cmd_list : list of str 858 A list of the components for the command line execution. Each 859 list entry is a single part of the command which is seperated from 860 the rest by a blank space. 861 E.g. ['mv', file1, file2] 862 A list of the components for the command line execution. 863 They will be concatenated with blank space for the command 864 to be submitted, like ['mv', file1, file2] for mv file1 file2. 862 865 863 866 Return 864 867 ------ 865 868 error_msg : str, optional 866 The possible error message if the subprocess failed.867 By default it will just tell"SUBPROCESS FAILED!".869 Error message if the subprocess fails. 870 By default it will just say "SUBPROCESS FAILED!". 868 871 ''' 869 872 … … 911 914 912 915 return start_period, end_period 916 917 918 def check_for_string_in_file(filepath, search_string): 919 """ 920 Search for a specific string in a file and return True if 921 the string was found. 922 923 Parameters 924 ---------- 925 filepath : str 926 The full file path which is to be examined. 927 928 search_string : str 929 The string which is looked up for in the file. 930 931 Return 932 ------ 933 Boolean : 934 True : String was found 935 False : String was not found 936 """ 937 with open(filepath, 'r') as fio: 938 for line in fio: 939 if search_string in line: 940 return True 941 return False 942 943 944 def overwrite_lines_in_file(filepath, search_string, sub_string): 945 """ 946 Overwrites lines which contain the given search string with the 947 substitution string. 948 949 Parameters 950 ---------- 951 search_string : str 952 The string which is looked up for in the file. 953 954 sub_string : str 955 The string which overwrites the search string. 956 957 Return 958 ------ 959 """ 960 with open(filepath, 'r') as fio: 961 data = fio.readlines() 962 963 with open(filepath, 'w') as fio: 964 for line in data: 965 if search_string in line: 966 fio.write(sub_string) 967 else: 968 fio.write(line) 969 970 return 971
Note: See TracChangeset
for help on using the changeset viewer.
