| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,121 +1,120 @@ | ||
| """ | ||
| Reads in the contributions.yaml file, and updates the entries by hitting the 'source' url. | ||
| """ | ||
|
|
||
| import argparse | ||
| from datetime import datetime, UTC | ||
| import pathlib | ||
| from ruamel.yaml import YAML | ||
| from multiprocessing import Pool | ||
|
|
||
| from parse_and_validate_properties_txt import read_properties_txt, parse_text, validate_existing | ||
| from scripts.parse_and_validate_properties_txt import ( | ||
| read_properties_txt, | ||
| parse_text, | ||
| validate_existing | ||
| ) | ||
|
|
||
|
|
||
| def update_contribution(contribution, props): | ||
| datetime_today = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%S%z') | ||
| contribution['lastUpdated'] = datetime_today | ||
| if 'previousVersions' not in contribution: | ||
| contribution['previousVersions'] = [] | ||
| contribution['previousVersions'].append(contribution['prettyVersion']) | ||
|
|
||
| # update from online | ||
| for field in props.keys(): | ||
| # process category list | ||
| if field == 'categories': | ||
| if props[field]: | ||
| contribution[field] = sorted(props[field].strip('"').split(',')) | ||
| else: | ||
| contribution[field] = [] | ||
| else: | ||
| contribution[field] = props[field] | ||
| datetime_today = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%S%z') | ||
|
Comment thread
Copy link
Copy Markdown
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Qualityapologies, this comment didn't get added to the previous review:
Sorry, something went wrong.
All reactions
|
||
| contribution['lastUpdated'] = datetime_today | ||
|
|
||
| if 'previousVersions' not in contribution: | ||
| contribution['previousVersions'] = [] | ||
|
|
||
| contribution['previousVersions'].append(contribution['prettyVersion']) | ||
|
|
||
| for field in props.keys(): | ||
| if field == 'categories': | ||
| if props[field]: | ||
| contribution[field] = sorted(props[field].strip('"').split(',')) | ||
| else: | ||
| contribution[field] = [] | ||
| else: | ||
| contribution[field] = props[field] | ||
|
|
||
| if 'download' not in contribution: | ||
| contribution['download'] = contribution['source'][:contribution['source'].rfind('.')] + '.zip' | ||
|
|
||
| if 'download' not in contribution: | ||
| contribution['download'] = contribution['source'][:contribution['source'].rfind('.')] + '.zip' | ||
|
|
||
|
|
||
| def log_broken(contribution, msg): | ||
| if contribution['status'] == 'VALID': | ||
| contribution['status'] = 'BROKEN' | ||
| if contribution['status'] == 'VALID': | ||
| contribution['status'] = 'BROKEN' | ||
|
|
||
| if 'log' not in contribution: | ||
| contribution['log'] = [] | ||
| contribution['log'] = [] | ||
|
|
||
| contribution['log'].append(msg) | ||
|
|
||
|
|
||
| def process_contribution(item): | ||
| index, contribution = item | ||
| index, contribution = item | ||
|
|
||
| date_today = datetime.now(UTC).strftime('%Y-%m-%d') | ||
| this_version = '0' | ||
| date_today = datetime.now(UTC).strftime('%Y-%m-%d') | ||
| this_version = '0' | ||
|
|
||
| if contribution['status'] != 'DEPRECATED': | ||
| # compare version to what is at url. If has changed, update contribution to | ||
| # what is online | ||
| if 'version' in contribution: | ||
| this_version = contribution['version'] | ||
| if contribution['status'] != 'DEPRECATED': | ||
| if 'version' in contribution: | ||
| this_version = contribution['version'] | ||
|
|
||
| try: | ||
| properties_raw = read_properties_txt(contribution['source']) | ||
| except FileNotFoundError as e: | ||
| log_broken(contribution, f'file not found, {e}, {date_today}') | ||
| return index, contribution | ||
| except Exception: | ||
| log_broken(contribution, f'url timeout, {date_today}') | ||
| return index, contribution | ||
| try: | ||
| properties_raw = read_properties_txt(contribution['source']) | ||
| except FileNotFoundError as e: | ||
| log_broken(contribution, f'file not found, {e}, {date_today}') | ||
| return index, contribution | ||
| except Exception: | ||
| log_broken(contribution, f'url timeout, {date_today}') | ||
| return index, contribution | ||
|
|
||
| try: | ||
| props = validate_existing(parse_text(properties_raw)) | ||
| except Exception: | ||
| log_broken(contribution, f'invalid file, {date_today}') | ||
| return index, contribution | ||
| try: | ||
| props = validate_existing(parse_text(properties_raw)) | ||
| except Exception: | ||
| log_broken(contribution, f'invalid file, {date_today}') | ||
| return index, contribution | ||
|
|
||
| # some library files have field lastUpdated. This also exists in the database, but is defined | ||
| # by our scripts, so remove this field. | ||
| contribution.pop('lastUpdated', None) | ||
| contribution.pop('lastUpdated', None) | ||
| contribution['status'] = 'VALID' | ||
|
|
||
| contribution['status'] = 'VALID' | ||
| if props['version'] != this_version: | ||
| update_contribution(contribution, props) | ||
|
|
||
| if props['version'] != this_version: | ||
| # update from online | ||
| update_contribution(contribution, props) | ||
| return index, contribution | ||
| return index, contribution | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| parser = argparse.ArgumentParser() | ||
| parser.add_argument('--index') | ||
| args = parser.parse_args() | ||
|
|
||
| index = 'all' | ||
| if args.index: | ||
| index = args.index | ||
|
|
||
| database_file = pathlib.Path(__file__).parent.parent / 'contributions.yaml' | ||
|
|
||
| # read in database yaml file | ||
| yaml = YAML() | ||
| with open(database_file, 'r') as db: | ||
| data = yaml.load(db) | ||
|
|
||
| contributions_list = data['contributions'] | ||
|
|
||
| if index == 'all': | ||
| total = len(contributions_list) | ||
| completed = 0 | ||
| print(f"Starting processing of {total} contributions...") | ||
|
|
||
| with Pool(processes=256) as pool: | ||
| for index, contribution in pool.imap_unordered(process_contribution, enumerate(contributions_list)): | ||
| contributions_list[index] = contribution | ||
| completed += 1 | ||
| print(f"Progress: {completed}/{total} ({(completed/total*100):.1f}%)") | ||
|
|
||
| print("All processing complete") | ||
| else: | ||
| # update only contribution with id==index | ||
| contribution = next((x for x in contributions_list if x['id'] == int(index)), None) | ||
| print(contribution) | ||
| process_contribution((index, contribution)) | ||
| print(contribution) | ||
|
|
||
| # write all contributions to database file | ||
| yaml = YAML() | ||
| with open(database_file, 'w') as outfile: | ||
| yaml.dump({"contributions": contributions_list}, outfile) | ||
| parser = argparse.ArgumentParser() | ||
| parser.add_argument('--index') | ||
| args = parser.parse_args() | ||
|
|
||
| index = 'all' | ||
| if args.index: | ||
| index = args.index | ||
|
|
||
| database_file = pathlib.Path(__file__).parent.parent / 'contributions.yaml' | ||
| yaml = YAML() | ||
|
|
||
| with open(database_file, 'r') as db: | ||
| data = yaml.load(db) | ||
|
|
||
| contributions_list = data['contributions'] | ||
|
|
||
| if index == 'all': | ||
| total = len(contributions_list) | ||
| completed = 0 | ||
| print(f"Starting processing of {total} contributions...") | ||
|
|
||
| with Pool(processes=256) as pool: | ||
| for index, contribution in pool.imap_unordered(process_contribution, enumerate(contributions_list)): | ||
| contributions_list[index] = contribution | ||
| completed += 1 | ||
| print(f"Progress: {completed}/{total} ({(completed/total*100):.1f}%)") | ||
|
|
||
| print("All processing complete") | ||
| else: | ||
| contribution = next((x for x in contributions_list if x['id'] == int(index)), None) | ||
| print(contribution) | ||
| process_contribution((index, contribution)) | ||
| print(contribution) | ||
|
|
||
| with open(database_file, 'w') as outfile: | ||
| yaml.dump({"contributions": contributions_list}, outfile) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| import os | ||
| import sys | ||
|
|
||
| # Allow imports from project root | ||
| sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) | ||
|
|
||
| from scripts import fetch_updates | ||
|
|
||
|
|
||
| def test_log_broken(): | ||
| contribution = { | ||
| "status": "VALID" | ||
| } | ||
|
|
||
| fetch_updates.log_broken(contribution, "file not found") | ||
|
|
||
| assert contribution["status"] == "BROKEN" | ||
| assert "log" in contribution | ||
| assert contribution["log"] == ["file not found"] | ||
|
|
||
|
|
||
| def test_update_contribution(): | ||
| contribution = { | ||
| "id": "test-id", | ||
| "prettyVersion": "1.0", | ||
| "source": "http://example.com/library.properties" | ||
| } | ||
|
|
||
| props = { | ||
| "version": "2.0", | ||
| "categories": '"Graphics,Animation"', | ||
| "author": "Test Author" | ||
| } | ||
|
|
||
| fetch_updates.update_contribution(contribution, props) | ||
|
|
||
| assert "lastUpdated" in contribution | ||
| assert contribution["previousVersions"] == ["1.0"] | ||
| assert contribution["version"] == "2.0" | ||
| assert contribution["categories"] == ["Animation", "Graphics"] | ||
| assert contribution["author"] == "Test Author" | ||
| assert contribution["download"].endswith(".zip") |
| Back | FazBrowse Home | New Git URL |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low QualityPlease leave in the comments that were previously there.
Sorry, something went wrong.
Uh oh!
There was an error while loading. Please reload this page.