Commit f927761
Con 140 flake8 (#110)
* Add linting to testing pipeline.
* black all .py files and ensuring they pass flake8
* Add some newlines, try to trigger build in travis.
* Try to rollback the change that broke the test
* Change is to == for a few lines
* Remove tests for deprecated error handling
* Remove == True1 parent 9d7395e commit f927761
25 files changed
Lines changed: 1152 additions & 1043 deletions
File tree
- codonPython
- tests
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,13 @@ | |||
| 1 | + [flake8] | ||
| 2 | + # Rule definitions: http://flake8.pycqa.org/en/latest/user/error-codes.html | ||
| 3 | + # D203: 1 blank line required before class docstring | ||
| 4 | + # W503: line break before binary operator | ||
| 5 | + # W504: line break after binary operator | ||
| 6 | + # F401: file imported but not used | ||
| 7 | + # F841: local variable is assigned to but never used | ||
| 8 | + exclude = __pycache__,node_modules,.git,.pytest_cache,docs | ||
| 9 | + ignore = D203,W503,W504,F401 | ||
| 10 | + max-complexity = 24 | ||
| 11 | + max-line-length = 120 | ||
| 12 | + per-file-ignores = | ||
| 13 | + codonPython/tests/file_utils_test.py:F841 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -10,6 +10,7 @@ install: | |||
| 10 | 10 | ||
| 11 | 11 | script: | |
| 12 | 12 | - pytest --cov=./ | |
| 13 | + - flake8 codonPython | ||
| 13 | 14 | ||
| 14 | 15 | after_success: | |
| 15 | 16 | - codecov | |
@@ -20,4 +21,4 @@ deploy: | |||
| 20 | 21 | skip_cleanup: true | |
| 21 | 22 | github_token: $githubtoken | |
| 22 | 23 | local_dir: docs/build/html | |
| 23 | - keep_history: true | ||
| 24 | + keep_history: true | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -5,7 +5,7 @@ def age_band_5_years(age: int) -> str: | |||
| 5 | 5 | """ | |
| 6 | 6 | Place age into appropriate 5 year band | |
| 7 | 7 | ||
| 8 | - This function takes the age supplied as an argument and returns a string | ||
| 8 | + This function takes the age supplied as an argument and returns a string | ||
| 9 | 9 | representing the relevant 5 year banding. | |
| 10 | 10 | ||
| 11 | 11 | Parameters | |
@@ -29,26 +29,26 @@ def age_band_5_years(age: int) -> str: | |||
| 29 | 29 | """ | |
| 30 | 30 | ||
| 31 | 31 | if age is None: | |
| 32 | - return 'Age not known' | ||
| 32 | + return "Age not known" | ||
| 33 | 33 | ||
| 34 | 34 | if age >= 90: | |
| 35 | 35 | if age >= 150: | |
| 36 | 36 | raise ValueError("The age input: {} is too large.".format(age)) | |
| 37 | 37 | else: | |
| 38 | - return '90 and over' | ||
| 38 | + return "90 and over" | ||
| 39 | 39 | elif age < 0: | |
| 40 | 40 | raise ValueError("The age input: {} is too low.".format(age)) | |
| 41 | 41 | else: | |
| 42 | 42 | lowerbound = 5 * int(math.floor(age / 5)) | |
| 43 | 43 | upperbound = lowerbound + 4 | |
| 44 | - return '{}-{}'.format(lowerbound, upperbound) | ||
| 44 | + return "{}-{}".format(lowerbound, upperbound) | ||
| 45 | 45 | ||
| 46 | 46 | ||
| 47 | 47 | def age_band_10_years(age: int) -> str: | |
| 48 | 48 | """ | |
| 49 | 49 | Place age into appropriate 10 year band | |
| 50 | 50 | ||
| 51 | - This function takes the age supplied as an argument and returns a string | ||
| 51 | + This function takes the age supplied as an argument and returns a string | ||
| 52 | 52 | representing the relevant 10 year banding. | |
| 53 | 53 | ||
| 54 | 54 | Parameters | |
@@ -72,16 +72,16 @@ def age_band_10_years(age: int) -> str: | |||
| 72 | 72 | """ | |
| 73 | 73 | ||
| 74 | 74 | if age is None: | |
| 75 | - return 'Age not known' | ||
| 75 | + return "Age not known" | ||
| 76 | 76 | ||
| 77 | 77 | if age >= 90: | |
| 78 | 78 | if age >= 150: | |
| 79 | 79 | raise ValueError("The age input: {} is too large.".format(age)) | |
| 80 | 80 | else: | |
| 81 | - return '90 and over' | ||
| 81 | + return "90 and over" | ||
| 82 | 82 | elif age < 0: | |
| 83 | 83 | raise ValueError("The age input: {} is too low.".format(age)) | |
| 84 | 84 | else: | |
| 85 | 85 | lowerbound = 10 * int(math.floor(age / 10)) | |
| 86 | 86 | upperbound = lowerbound + 9 | |
| 87 | - return '{}-{}'.format(lowerbound, upperbound) | ||
| 87 | + return "{}-{}".format(lowerbound, upperbound) | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -2,7 +2,12 @@ | |||
| 2 | 2 | import numpy as np | |
| 3 | 3 | ||
| 4 | 4 | ||
| 5 | - def check_consistent_measures(data, geography_col: str = "Org_Level", measure_col: str = "Measure", measures_set: set = set()) -> bool: | ||
| 5 | + def check_consistent_measures( | ||
| 6 | + data, | ||
| 7 | + geography_col: str = "Org_Level", | ||
| 8 | + measure_col: str = "Measure", | ||
| 9 | + measures_set: set = set(), | ||
| 10 | + ) -> bool: | ||
| 6 | 11 | """ | |
| 7 | 12 | Check every measure is in every geography level. | |
| 8 | 13 | ||
@@ -48,7 +53,7 @@ def check_consistent_measures(data, geography_col: str = "Org_Level", measure_co | |||
| 48 | 53 | ||
| 49 | 54 | if data.isna().any(axis=None): | |
| 50 | 55 | raise ValueError( | |
| 51 | - f"Missing values at locations {list(map(tuple, np.argwhere(data.isna().values)))}" | ||
| 56 | + f"Missing values at locations {list(map(tuple, np.argwhere(data.isna().values)))}" | ||
| 52 | 57 | ) | |
| 53 | 58 | if not isinstance(geography_col, str) or not isinstance(measure_col, str): | |
| 54 | 59 | raise ValueError("Please input strings for column indexes.") | |
@@ -59,8 +64,7 @@ def check_consistent_measures(data, geography_col: str = "Org_Level", measure_co | |||
| 59 | 64 | ||
| 60 | 65 | # Every geography level should have the same set of measures as the global set. | |
| 61 | 66 | global_set = measures_set if measures_set else set(data[measure_col].unique()) | |
| 62 | - subsets = data.groupby(geography_col) \ | ||
| 63 | - .agg({measure_col: "unique"}) | ||
| 67 | + subsets = data.groupby(geography_col).agg({measure_col: "unique"}) | ||
| 64 | 68 | subset_agreement = all(set(x) == global_set for x in subsets[measure_col]) | |
| 65 | 69 | ||
| 66 | 70 | return subset_agreement | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,7 +1,13 @@ | |||
| 1 | 1 | import pandas as pd | |
| 2 | 2 | ||
| 3 | 3 | ||
| 4 | - def check_consistent_submissions(data, national_geog_level: str = "National", geography_col: str = "Org_Level", submissions_col: str = "Value_Unsuppressed", measure_col: str = "Measure", ) -> bool: | ||
| 4 | + def check_consistent_submissions( | ||
| 5 | + data, | ||
| 6 | + national_geog_level: str = "National", | ||
| 7 | + geography_col: str = "Org_Level", | ||
| 8 | + submissions_col: str = "Value_Unsuppressed", | ||
| 9 | + measure_col: str = "Measure", | ||
| 10 | + ) -> bool: | ||
| 5 | 11 | """ | |
| 6 | 12 | Check total submissions for each measure are the same across all geography levels | |
| 7 | 13 | except national. | |
@@ -49,24 +55,28 @@ def check_consistent_submissions(data, national_geog_level: str = "National", ge | |||
| 49 | 55 | """ | |
| 50 | 56 | ||
| 51 | 57 | if ( | |
| 52 | - not isinstance(submissions_col, str) or | ||
| 53 | - not isinstance(measure_col, str) or | ||
| 54 | - not isinstance(geography_col, str) or | ||
| 55 | - not isinstance(national_geog_level, str) | ||
| 58 | + not isinstance(submissions_col, str) | ||
| 59 | + or not isinstance(measure_col, str) | ||
| 60 | + or not isinstance(geography_col, str) | ||
| 61 | + or not isinstance(national_geog_level, str) | ||
| 56 | 62 | ): | |
| 57 | - raise ValueError("Please input strings for column names and national geography level.") | ||
| 63 | + raise ValueError( | ||
| 64 | + "Please input strings for column names and national geography level." | ||
| 65 | + ) | ||
| 58 | 66 | if ( | |
| 59 | - submissions_col not in data.columns or | ||
| 60 | - measure_col not in data.columns or | ||
| 61 | - geography_col not in data.columns | ||
| 67 | + submissions_col not in data.columns | ||
| 68 | + or measure_col not in data.columns | ||
| 69 | + or geography_col not in data.columns | ||
| 62 | 70 | ): | |
| 63 | 71 | raise KeyError("Check column names correspond to the DataFrame.") | |
| 64 | 72 | ||
| 65 | 73 | # All non-national measures should have only one unique submission number for each | |
| 66 | 74 | # geography level. | |
| 67 | - submissions_by_measure = data[data[geography_col] != national_geog_level] \ | ||
| 68 | - .groupby(measure_col) \ | ||
| 69 | - .agg({submissions_col: "nunique"}) | ||
| 75 | + submissions_by_measure = ( | ||
| 76 | + data[data[geography_col] != national_geog_level] | ||
| 77 | + .groupby(measure_col) | ||
| 78 | + .agg({submissions_col: "nunique"}) | ||
| 79 | + ) | ||
| 70 | 80 | result = (submissions_by_measure[submissions_col] == 1).all() | |
| 71 | 81 | ||
| 72 | 82 | return result | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,9 +1,13 @@ | |||
| 1 | 1 | import pandas as pd | |
| 2 | 2 | ||
| 3 | 3 | ||
| 4 | - def check_nat_val(df: pd.DataFrame, breakdown_col: str = "Breakdown", | ||
| 5 | - measure_col: str = "Measure", value_col: str = | ||
| 6 | - "Value_Unsuppressed", nat_val: str = "National") -> bool: | ||
| 4 | + def check_nat_val( | ||
| 5 | + df: pd.DataFrame, | ||
| 6 | + breakdown_col: str = "Breakdown", | ||
| 7 | + measure_col: str = "Measure", | ||
| 8 | + value_col: str = "Value_Unsuppressed", | ||
| 9 | + nat_val: str = "National", | ||
| 10 | + ) -> bool: | ||
| 7 | 11 | """ | |
| 8 | 12 | Check national value less than or equal to sum of breakdowns. | |
| 9 | 13 | ||
@@ -66,24 +70,32 @@ def check_nat_val(df: pd.DataFrame, breakdown_col: str = "Breakdown", | |||
| 66 | 70 | False | |
| 67 | 71 | """ | |
| 68 | 72 | ||
| 69 | - if not isinstance(breakdown_col, str) or not isinstance(measure_col, str)\ | ||
| 70 | - or not isinstance(value_col, str): | ||
| 73 | + if ( | ||
| 74 | + not isinstance(breakdown_col, str) | ||
| 75 | + or not isinstance(measure_col, str) | ||
| 76 | + or not isinstance(value_col, str) | ||
| 77 | + ): | ||
| 71 | 78 | raise ValueError("Please input strings for column indexes.") | |
| 72 | 79 | if not isinstance(nat_val, str): | |
| 73 | 80 | raise ValueError("Please input strings for value indexes.") | |
| 74 | - if breakdown_col not in df.columns or measure_col not in df.columns or\ | ||
| 75 | - value_col not in df.columns: | ||
| 81 | + if ( | ||
| 82 | + breakdown_col not in df.columns | ||
| 83 | + or measure_col not in df.columns | ||
| 84 | + or value_col not in df.columns | ||
| 85 | + ): | ||
| 76 | 86 | raise KeyError("Check column names correspond to the DataFrame.") | |
| 77 | - # aggregate values by measure and breakdown | ||
| 78 | - grouped = df.groupby([measure_col, breakdown_col]).agg({value_col: sum})\ | ||
| 79 | - .reset_index() | ||
| 87 | + # aggregate values by measure and breakdown | ||
| 88 | + grouped = ( | ||
| 89 | + df.groupby([measure_col, breakdown_col]).agg({value_col: sum}).reset_index() | ||
| 90 | + ) | ||
| 80 | 91 | national = grouped.loc[grouped[breakdown_col] == nat_val].reset_index() | |
| 81 | 92 | non_national = grouped.loc[grouped[breakdown_col] != nat_val].reset_index() | |
| 82 | - # check values are less than or equal to national value for each measure | ||
| 83 | - join = pd.merge(non_national, national, left_on=measure_col, | ||
| 84 | - right_on=measure_col, how='left') | ||
| 85 | - left = value_col + '_x' | ||
| 86 | - right = value_col + '_y' | ||
| 87 | - join['Check'] = join[right] <= join[left] | ||
| 88 | - result = all(join['Check']) | ||
| 93 | + # check values are less than or equal to national value for each measure | ||
| 94 | + join = pd.merge( | ||
| 95 | + non_national, national, left_on=measure_col, right_on=measure_col, how="left" | ||
| 96 | + ) | ||
| 97 | + left = value_col + "_x" | ||
| 98 | + right = value_col + "_y" | ||
| 99 | + join["Check"] = join[right] <= join[left] | ||
| 100 | + result = all(join["Check"]) | ||
| 89 | 101 | return result | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,11 +1,13 @@ | |||
| 1 | 1 | import numpy | |
| 2 | 2 | import pandas as pd | |
| 3 | 3 | ||
| 4 | + | ||
| 4 | 5 | def check_null(dataframe: pd.DataFrame, columns_to_be_checked: list) -> int: | |
| 5 | 6 | """ | |
| 6 | 7 | Checks a pandas dataframe for null values | |
| 7 | 8 | ||
| 8 | - This function takes a pandas dataframe supplied as an argument and returns a integer value representing any null values found within the columns to check | ||
| 9 | + This function takes a pandas dataframe supplied as an argument and returns a integer value | ||
| 10 | + representing any null values found within the columns to check. | ||
| 9 | 11 | ||
| 10 | 12 | Parameters | |
| 11 | 13 | ---------- | |
@@ -29,18 +31,16 @@ def check_null(dataframe: pd.DataFrame, columns_to_be_checked: list) -> int: | |||
| 29 | 31 | ||
| 30 | 32 | if not isinstance(columns_to_be_checked, list): | |
| 31 | 33 | raise ValueError("Please make sure that all your columns passed are strings") | |
| 32 | - else: | ||
| 33 | - pass | ||
| 34 | 34 | ||
| 35 | 35 | for eachCol in columns_to_be_checked: | |
| 36 | 36 | if eachCol not in dataframe.columns: | |
| 37 | - raise KeyError("Please check the column names correspond to values in the DataFrame.") | ||
| 38 | - else: | ||
| 39 | - pass | ||
| 37 | + raise KeyError( | ||
| 38 | + "Please check the column names correspond to values in the DataFrame." | ||
| 39 | + ) | ||
| 40 | 40 | ||
| 41 | 41 | null_count = 0 | |
| 42 | 42 | for eachColumn in columns_to_be_checked: | |
| 43 | 43 | prev_null_count = null_count | |
| 44 | 44 | null_count = prev_null_count + (len(dataframe) - dataframe[eachColumn].count()) | |
| 45 | 45 | ||
| 46 | - return null_count | ||
| 46 | + return null_count | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -3,7 +3,7 @@ | |||
| 3 | 3 | ||
| 4 | 4 | def validDate(date_string: str) -> bool: | |
| 5 | 5 | """ | |
| 6 | - Validates stringtype dates of type `dd/mm/yyyy`, `dd-mm-yyyy` or `dd.mm.yyyy` from | ||
| 6 | + Validates stringtype dates of type `dd/mm/yyyy`, `dd-mm-yyyy` or `dd.mm.yyyy` from | ||
| 7 | 7 | years 1900-9999. Leap year support included. | |
| 8 | 8 | ||
| 9 | 9 | Parameters | |
@@ -33,13 +33,15 @@ def validDate(date_string: str) -> bool: | |||
| 33 | 33 | # https://stackoverflow.com/questions/15491894/regex-to-validate-date-format-dd-mm-yyyy | |
| 34 | 34 | # modified to confine the year dates. | |
| 35 | 35 | if re.match( | |
| 36 | - r"^(?:(?:31(\/|-|\.)(?:0?[13578]|1[02]))\1" + | ||
| 37 | - r"|(?:(?:29|30)(\/|-|\.)(?:0?[13-9]|1[0-2])\2" + | ||
| 38 | - r"))(?:(?:1[9]..|2[0][0-4].))$|^(?:29(\/|-|\.)0?2\3" + | ||
| 39 | - r"(?:(?:(?:1[6-9]|[2-9]\d)?(?:0[48]|[2468][048]|[13579][26])|(?:(?:16|[2468][048]" + | ||
| 40 | - r"|[3579][26])00))))$|^(?:0?[1-9]|1\d|2[0-8])(\/|-|\.)(?:(?:0?[1-9])|(?:1[0-2]))\4" + | ||
| 41 | - r"(?:(?:1[9]..|2[0][0-4].))$", | ||
| 42 | - date_string, flags=0): | ||
| 36 | + r"^(?:(?:31(\/|-|\.)(?:0?[13578]|1[02]))\1" | ||
| 37 | + + r"|(?:(?:29|30)(\/|-|\.)(?:0?[13-9]|1[0-2])\2" | ||
| 38 | + + r"))(?:(?:1[9]..|2[0][0-4].))$|^(?:29(\/|-|\.)0?2\3" | ||
| 39 | + + r"(?:(?:(?:1[6-9]|[2-9]\d)?(?:0[48]|[2468][048]|[13579][26])|(?:(?:16|[2468][048]" | ||
| 40 | + + r"|[3579][26])00))))$|^(?:0?[1-9]|1\d|2[0-8])(\/|-|\.)(?:(?:0?[1-9])|(?:1[0-2]))\4" | ||
| 41 | + + r"(?:(?:1[9]..|2[0][0-4].))$", | ||
| 42 | + date_string, | ||
| 43 | + flags=0, | ||
| 44 | + ): | ||
| 43 | 45 | return True | |
| 44 | 46 | else: | |
| 45 | 47 | return False | |
0 commit comments