FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

chore(version-scanner): configure GHA to use targets file for multi-v… · googleapis/google-cloud-python@204667b · GitHub

/ google-cloud-python Public

Commit 204667b

Browse files
chalmerlowe
authored
chore(version-scanner): configure GHA to use targets file for multi-version scanning (#17538)
This PR configures the Automated Dependency Version Scanner GHA workflow to use a YAML targets file instead of hardcoded dependency/version parameters, and limits scanning to the 31 handwritten and hybrid packages for speed.
1 parent a5ad18c commit 204667b

6 files changed

Lines changed: 101 additions & 42 deletions

File tree

‎.github/workflows/version_scanner.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ jobs:
3535
# Uses -o to output a detailed, raw CSV to a file
3636
# Uses --stdout to print a slim, easier to parse summary to the GitHub Actions UI
3737
# Uses --soft-fail to temporarily limit causing CI/CD failures during the migration to full operation.
38-
python scripts/version_scanner/version_scanner.py -d python -v 3.7 --stdout -o version_scanner_output.csv --soft-fail
38+
python scripts/version_scanner/version_scanner.py --matrix-file scripts/version_scanner/matrix.yaml --package-file scripts/version_scanner/example-list-non-generated-packages.txt --stdout -o version_scanner_output.csv --soft-fail
3939
4040
- name: Upload CSV Results
4141
if: always()

‎scripts/version_scanner/example-list-non-generated-packages.txt‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
packages/bigframes
2+
packages/bigquery-magics
3+
packages/db-dtypes
4+
packages/django-google-spanner
5+
packages/gapic-generator
6+
packages/google-api-core
7+
# packages/google-api-python-client # non-monorepo, ignore for now.
8+
packages/google-auth
9+
packages/google-auth-httplib2
10+
packages/google-auth-oauthlib
11+
packages/google-cloud-bigquery
12+
packages/pandas-gbq
13+
packages/google-cloud-bigtable
14+
packages/google-cloud-core
15+
packages/google-crc32c
16+
packages/google-cloud-datastore
17+
packages/google-cloud-dns
18+
packages/google-cloud-documentai-toolbox
19+
packages/google-cloud-error-reporting
20+
packages/google-cloud-firestore
21+
packages/google-cloud-logging
22+
packages/google-cloud-ndb
23+
packages/google-cloud-pubsub
24+
packages/google-cloud-runtimeconfig
25+
packages/google-cloud-spanner
26+
packages/google-cloud-storage
27+
packages/google-cloud-testutils
28+
packages/google-resumable-media
29+
packages/proto-plus
30+
packages/sqlalchemy-bigquery
31+
packages/sqlalchemy-spanner

‎scripts/version_scanner/matrix.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
python:
2+
- "3.7"
3+
- "3.8"
4+
- "3.9"

‎scripts/version_scanner/small_package_list.txt‎

Lines changed: 0 additions & 6 deletions
This file was deleted.

‎scripts/version_scanner/tests/unit/test_version_scanner.py‎

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ def sample_match():
5959
(PermissionError(), False, False, False, "Warning: Permission denied reading test_desc", None), # Optional PermissionError
6060
(IOError("disk full"), True, False, True, "Error reading test_desc", None), # Required IOError
6161
(IOError("disk full"), False, False, False, "Warning: Error reading test_desc", None), # Optional IOError
62+
(ValueError("invalid bytes"), True, False, True, "Error reading test_desc", None), # Required ValueError
63+
(ValueError("invalid bytes"), False, False, False, "Warning: Error reading test_desc", None), # Optional ValueError
6264
]
6365
)
6466
def test_safe_read_file_scenarios(
@@ -782,16 +784,16 @@ def test_format_for_console(sample_match):
782784
assert "python_requires = " not in log_str # Slim format doesn't print context line
783785

784786

785-
def test_parse_targets_file(tmp_path):
786-
from version_scanner import parse_targets_file
787-
yaml_file = tmp_path / "targets.yaml"
787+
def test_parse_matrix_file(tmp_path):
788+
from version_scanner import parse_matrix_file
789+
yaml_file = tmp_path / "matrix.yaml"
788790
yaml_file.write_text("""
789791
python:
790792
- "3.7"
791793
- "3.8"
792794
protobuf: "4.25.8"
793795
""")
794-
targets = parse_targets_file(str(yaml_file))
796+
targets = parse_matrix_file(str(yaml_file))
795797
assert targets == [("python", "3.7"), ("python", "3.8"), ("protobuf", "4.25.8")]
796798

797799
@pytest.mark.parametrize(
@@ -801,20 +803,22 @@ def test_parse_targets_file(tmp_path):
801803
("invalid: {", True), # Invalid YAML
802804
("- not_a_mapping", True), # Invalid structure (list instead of map)
803805
("python:\n - null", True), # Invalid version type (null/None value)
806+
("python:\n - 3.10", True), # Invalid version type (float instead of string in list)
807+
("python: 3.10", True), # Invalid version type (float instead of string)
804808
]
805809
)
806-
def test_parse_targets_file_failures(tmp_path, file_content, file_exists):
807-
from version_scanner import parse_targets_file
810+
def test_parse_matrix_file_failures(tmp_path, file_content, file_exists):
811+
from version_scanner import parse_matrix_file
808812

809813
if file_exists:
810-
yaml_file = tmp_path / "targets_failures.yaml"
814+
yaml_file = tmp_path / "matrix_failures.yaml"
811815
yaml_file.write_text(file_content)
812816
path = str(yaml_file)
813817
else:
814818
path = "nonexistent_file.yaml"
815819

816820
with pytest.raises(SystemExit) as excinfo:
817-
parse_targets_file(path)
821+
parse_matrix_file(path)
818822
assert excinfo.value.code == 1
819823

820824
def test_scan_repository_multi_targets(tmp_path):
@@ -868,3 +872,26 @@ def test_scan_repository_multi_targets(tmp_path):
868872
assert protobuf_match[0]["version"] == "4.25.8"
869873
assert protobuf_match[0]["rule_name"] == "protobuf_check"
870874

875+
876+
@pytest.mark.parametrize(
877+
"args, expected_error_msg",
878+
[
879+
# Mixing -m/--matrix-file with -d or -v
880+
(['version_scanner.py', '-m', 'matrix.yaml', '-d', 'python'], "Cannot specify -d/--dependency or -v/--version when using -m/--matrix-file"),
881+
(['version_scanner.py', '-m', 'matrix.yaml', '-v', '3.7'], "Cannot specify -d/--dependency or -v/--version when using -m/--matrix-file"),
882+
(['version_scanner.py', '-m', 'matrix.yaml', '-d', 'python', '-v', '3.7'], "Cannot specify -d/--dependency or -v/--version when using -m/--matrix-file"),
883+
# Missing either -d or -v when not using -m
884+
(['version_scanner.py', '-d', 'python'], "Must specify both -d/--dependency and -v/--version when not using -m/--matrix-file"),
885+
(['version_scanner.py', '-v', '3.7'], "Must specify both -d/--dependency and -v/--version when not using -m/--matrix-file"),
886+
(['version_scanner.py'], "Must specify both -d/--dependency and -v/--version when not using -m/--matrix-file"),
887+
]
888+
)
889+
def test_main_cli_validation(capsys, args, expected_error_msg):
890+
from version_scanner import main
891+
with mock.patch('sys.argv', args):
892+
with pytest.raises(SystemExit) as excinfo:
893+
main()
894+
assert excinfo.value.code == 2
895+
captured = capsys.readouterr()
896+
assert expected_error_msg in captured.err
897+

‎scripts/version_scanner/version_scanner.py‎

Lines changed: 30 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ def _safe_read_file(
6565
else:
6666
print(f"Warning: Permission denied reading {description}: {file_path}", file=sys.stderr)
6767
return None
68-
except IOError as e:
68+
except (IOError, ValueError) as e:
6969
if required:
7070
print(f"Error reading {description} {file_path}: {e}", file=sys.stderr)
7171
sys.exit(1)
@@ -624,33 +624,36 @@ def scan_repository(
624624
return results
625625

626626

627-
def parse_targets_file(file_path: str) -> List[Tuple[str, str]]:
627+
def parse_matrix_file(file_path: str) -> List[Tuple[str, str]]:
628628
"""
629-
Parses a YAML targets file into a list of (dependency, version) tuples.
629+
Parses a YAML matrix file into a list of (dependency, version) tuples.
630630
"""
631-
content = _safe_read_file(file_path, required=True, description="targets file")
631+
content = _safe_read_file(file_path, required=True, description="matrix file")
632632
try:
633-
raw_targets = yaml.safe_load(content)
633+
raw_matrix = yaml.safe_load(content)
634634
except Exception as e:
635-
print(f"Error parsing targets YAML mapping: {e}", file=sys.stderr)
635+
print(f"Error parsing matrix YAML mapping: {e}", file=sys.stderr)
636636
sys.exit(1)
637637

638-
if not isinstance(raw_targets, dict):
639-
print("Error: Targets file content must resolve to a YAML mapping", file=sys.stderr)
638+
if not isinstance(raw_matrix, dict):
639+
print("Error: Matrix file content must resolve to a YAML mapping", file=sys.stderr)
640640
sys.exit(1)
641641

642642
targets = []
643-
for dep, versions in raw_targets.items():
643+
for dep, versions in raw_matrix.items():
644644
if isinstance(versions, list):
645645
for v in versions:
646646
if v is None or isinstance(v, (dict, list)):
647647
print(f"Error: Invalid version '{v}' for dependency '{dep}'", file=sys.stderr)
648648
sys.exit(1)
649-
targets.append((str(dep), str(v)))
650-
elif versions is not None and not isinstance(versions, dict):
651-
targets.append((str(dep), str(versions)))
649+
if not isinstance(v, str):
650+
print(f"Error: Version '{v}' for dependency '{dep}' must be specified as a quoted string to prevent YAML parsing issues (e.g., 3.10 parsed as 3.1).", file=sys.stderr)
651+
sys.exit(1)
652+
targets.append((str(dep), v))
653+
elif isinstance(versions, str):
654+
targets.append((str(dep), versions))
652655
else:
653-
print(f"Error: Invalid version '{versions}' for dependency '{dep}'", file=sys.stderr)
656+
print(f"Error: Invalid version '{versions}' for dependency '{dep}'. Versions must be specified as quoted strings.", file=sys.stderr)
654657
sys.exit(1)
655658

656659
return targets
@@ -675,7 +678,7 @@ def main():
675678
)
676679

677680
parser.add_argument(
678-
"--targets-file",
681+
"-m", "--matrix-file",
679682
help="Path to a YAML file containing target dependencies and versions."
680683
)
681684

@@ -743,17 +746,17 @@ def main():
743746
args = parser.parse_args()
744747

745748
# Validation of required inputs
746-
has_single_target = bool(args.dependency and args.version)
747-
has_targets_file = bool(args.targets_file)
748-
749-
if not (has_single_target or has_targets_file):
750-
parser.error("Must specify either (-d/--dependency AND -v/--version) OR (--targets-file)")
751-
if has_single_target and has_targets_file:
752-
parser.error("Cannot specify both single target (-d/-v) and targets file (--targets-file)")
749+
has_matrix_file = bool(args.matrix_file)
750+
if has_matrix_file:
751+
if args.dependency or args.version:
752+
parser.error("Cannot specify -d/--dependency or -v/--version when using -m/--matrix-file")
753+
else:
754+
if not (args.dependency and args.version):
755+
parser.error("Must specify both -d/--dependency and -v/--version when not using -m/--matrix-file")
753756

754757
targets = []
755-
if has_targets_file:
756-
targets = parse_targets_file(args.targets_file)
758+
if has_matrix_file:
759+
targets = parse_matrix_file(args.matrix_file)
757760
else:
758761
targets = [(args.dependency, args.version)]
759762

@@ -772,7 +775,7 @@ def main():
772775
elif args.package_file:
773776
target_packages = read_package_file(args.package_file)
774777

775-
if has_targets_file:
778+
if has_matrix_file:
776779
print("Starting scan for multiple targets:")
777780
for dep, ver in targets:
778781
print(f" - {dep}: {ver}")
@@ -809,7 +812,7 @@ def main():
809812
rules,
810813
target_packages,
811814
ignore_dirs,
812-
version_string=(None if has_targets_file else args.version),
815+
version_string=(None if has_matrix_file else args.version),
813816
targets=targets
814817
)
815818

@@ -833,8 +836,8 @@ def main():
833836
script_dir = os.path.dirname(os.path.abspath(__file__))
834837
results_dir = os.path.join(script_dir, "results")
835838
os.makedirs(results_dir, exist_ok=True)
836-
if has_targets_file:
837-
base_name = os.path.splitext(os.path.basename(args.targets_file))[0]
839+
if has_matrix_file:
840+
base_name = os.path.splitext(os.path.basename(args.matrix_file))[0]
838841
output_path = os.path.join(results_dir, f"{base_name}-{timestamp}.csv")
839842
else:
840843
output_path = os.path.join(results_dir, f"{args.dependency}-{args.version}-{timestamp}.csv")

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL