| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,60 +1,84 @@ | |||
| 1 | 1 | import os | |
| 2 | 2 | ||
| 3 | - from kaggle_gcp import get_integrations | ||
| 4 | 3 | from log import Log | |
| 5 | 4 | ||
| 6 | 5 | kaggle_proxy_token = os.getenv("KAGGLE_DATA_PROXY_TOKEN") | |
| 7 | 6 | kernel_integrations_var = os.getenv("KAGGLE_KERNEL_INTEGRATIONS") | |
| 8 | 7 | ||
| 9 | 8 | def init(): | |
| 10 | - bq_user_jwt = os.getenv("KAGGLE_USER_SECRETS_TOKEN") | ||
| 11 | - if kaggle_proxy_token or bq_user_jwt: | ||
| 12 | - from google.auth import credentials, environment_vars | ||
| 13 | - from google.cloud import bigquery | ||
| 14 | - from google.cloud.bigquery._http import Connection | ||
| 15 | - # TODO: Update this to the correct kaggle.gcp path once we no longer inject modules | ||
| 16 | - # from the worker. | ||
| 17 | - from kaggle_gcp import PublicBigqueryClient, KaggleKernelCredentials | ||
| 18 | - | ||
| 19 | - # If this Kernel has bigquery integration on startup, preload the Kaggle Credentials | ||
| 20 | - # object for magics to work. | ||
| 21 | - if get_integrations().has_bigquery(): | ||
| 22 | - from google.cloud.bigquery import magics | ||
| 23 | - magics.context.credentials = KaggleKernelCredentials() | ||
| 24 | - | ||
| 25 | - def monkeypatch_bq(bq_client, *args, **kwargs): | ||
| 26 | - specified_credentials = kwargs.get('credentials') | ||
| 27 | - has_bigquery = get_integrations().has_bigquery() | ||
| 28 | - # Prioritize passed in project id, but if it is missing look for env var. | ||
| 29 | - arg_project = kwargs.get('project') | ||
| 30 | - explicit_project_id = arg_project or os.environ.get(environment_vars.PROJECT) | ||
| 31 | - # This is a hack to get around the bug in google-cloud library. | ||
| 32 | - # Remove these two lines once this is resolved: | ||
| 33 | - # https://github.com/googleapis/google-cloud-python/issues/8108 | ||
| 34 | - if explicit_project_id: | ||
| 35 | - Log.info(f"Explicit project set to {explicit_project_id}") | ||
| 36 | - kwargs['project'] = explicit_project_id | ||
| 37 | - if explicit_project_id is None and specified_credentials is None and not has_bigquery: | ||
| 38 | - msg = "Using Kaggle's public dataset BigQuery integration." | ||
| 39 | - Log.info(msg) | ||
| 40 | - print(msg) | ||
| 41 | - return PublicBigqueryClient(*args, **kwargs) | ||
| 42 | - | ||
| 43 | - else: | ||
| 44 | - if specified_credentials is None: | ||
| 45 | - Log.info("No credentials specified, using KaggleKernelCredentials.") | ||
| 46 | - kwargs['credentials'] = KaggleKernelCredentials() | ||
| 47 | - if (not has_bigquery): | ||
| 48 | - Log.info("No bigquery integration found, creating client anyways.") | ||
| 49 | - print('Please ensure you have selected a BigQuery ' | ||
| 50 | - 'account in the Kernels Settings sidebar.') | ||
| 51 | - return bq_client(*args, **kwargs) | ||
| 52 | - | ||
| 53 | - # Monkey patches BigQuery client creation to use proxy or user-connected GCP account. | ||
| 54 | - # Deprecated in favor of Kaggle.DataProxyClient(). | ||
| 55 | - # TODO: Remove this once uses have migrated to that new interface. | ||
| 56 | - bq_client = bigquery.Client | ||
| 57 | - bigquery.Client = lambda *args, **kwargs: monkeypatch_bq( | ||
| 58 | - bq_client, *args, **kwargs) | ||
| 9 | + is_jwe_set = "KAGGLE_USER_SECRETS_TOKEN" in os.environ | ||
| 10 | + if kaggle_proxy_token or is_jwe_set: | ||
| 11 | + init_bigquery() | ||
| 12 | + if is_jwe_set: | ||
| 13 | + if get_integrations().has_gcs(): | ||
| 14 | + init_gcs() | ||
| 15 | + | ||
| 16 | + | ||
| 17 | + def init_bigquery(): | ||
| 18 | + from google.auth import credentials, environment_vars | ||
| 19 | + from google.cloud import bigquery | ||
| 20 | + from google.cloud.bigquery._http import Connection | ||
| 21 | + # TODO: Update this to the correct kaggle.gcp path once we no longer inject modules | ||
| 22 | + # from the worker. | ||
| 23 | + from kaggle_gcp import get_integrations, PublicBigqueryClient, KaggleKernelCredentials | ||
| 24 | + | ||
| 25 | + # If this Kernel has bigquery integration on startup, preload the Kaggle Credentials | ||
| 26 | + # object for magics to work. | ||
| 27 | + if get_integrations().has_bigquery(): | ||
| 28 | + from google.cloud.bigquery import magics | ||
| 29 | + magics.context.credentials = KaggleKernelCredentials() | ||
| 30 | + | ||
| 31 | + def monkeypatch_bq(bq_client, *args, **kwargs): | ||
| 32 | + specified_credentials = kwargs.get('credentials') | ||
| 33 | + has_bigquery = get_integrations().has_bigquery() | ||
| 34 | + # Prioritize passed in project id, but if it is missing look for env var. | ||
| 35 | + arg_project = kwargs.get('project') | ||
| 36 | + explicit_project_id = arg_project or os.environ.get(environment_vars.PROJECT) | ||
| 37 | + # This is a hack to get around the bug in google-cloud library. | ||
| 38 | + # Remove these two lines once this is resolved: | ||
| 39 | + # https://github.com/googleapis/google-cloud-python/issues/8108 | ||
| 40 | + if explicit_project_id: | ||
| 41 | + Log.info(f"Explicit project set to {explicit_project_id}") | ||
| 42 | + kwargs['project'] = explicit_project_id | ||
| 43 | + if explicit_project_id is None and specified_credentials is None and not has_bigquery: | ||
| 44 | + msg = "Using Kaggle's public dataset BigQuery integration." | ||
| 45 | + Log.info(msg) | ||
| 46 | + print(msg) | ||
| 47 | + return PublicBigqueryClient(*args, **kwargs) | ||
| 48 | + | ||
| 49 | + else: | ||
| 50 | + if specified_credentials is None: | ||
| 51 | + Log.info("No credentials specified, using KaggleKernelCredentials.") | ||
| 52 | + kwargs['credentials'] = KaggleKernelCredentials() | ||
| 53 | + if (not has_bigquery): | ||
| 54 | + Log.info("No bigquery integration found, creating client anyways.") | ||
| 55 | + print('Please ensure you have selected a BigQuery ' | ||
| 56 | + 'account in the Kernels Settings sidebar.') | ||
| 57 | + return bq_client(*args, **kwargs) | ||
| 58 | + | ||
| 59 | + # Monkey patches BigQuery client creation to use proxy or user-connected GCP account. | ||
| 60 | + # Deprecated in favor of Kaggle.DataProxyClient(). | ||
| 61 | + # TODO: Remove this once uses have migrated to that new interface. | ||
| 62 | + bq_client = bigquery.Client | ||
| 63 | + bigquery.Client = lambda *args, **kwargs: monkeypatch_bq( | ||
| 64 | + bq_client, *args, **kwargs) | ||
| 65 | + | ||
| 66 | + | ||
| 67 | + def init_gcs(): | ||
| 68 | + from kaggle_gcp import get_integrations | ||
| 69 | + if get_integrations().has_gcs(): | ||
| 70 | + from kaggle_secrets import GcpTarget | ||
| 71 | + from kaggle_gcp import KaggleKernelCredentials | ||
| 72 | + from google.cloud import storage | ||
| 73 | + def monkeypatch_gcs(gcs_client, *args, **kwargs): | ||
| 74 | + specified_credentials = kwargs.get('credentials') | ||
| 75 | + if specified_credentials is None: | ||
| 76 | + Log.info("No credentials specified, using KaggleKernelCredentials.") | ||
| 77 | + kwargs['credentials'] = KaggleKernelCredentials(target=GcpTarget.GCS) | ||
| 78 | + return gcs_client(*args, **kwargs) | ||
| 79 | + | ||
| 80 | + gcs_client = storage.Client | ||
| 81 | + storage.Client = lambda *args, **kwargs: monkeypatch_gcs(gcs_client, *args, **kwargs) | ||
| 82 | + | ||
| 59 | 83 | ||
| 60 | 84 | init() | |
| Back | FazBrowse Home | New Git URL |
0 commit comments