FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Set KaggleKernelCredentials as default for GCS · AI-For-Rural/docker-python@44f637a · GitHub

Repository navigation

Commit 44f637a

Browse files
committed
Set KaggleKernelCredentials as default for GCS
1 parent 2ac2889 commit 44f637a

1 file changed

Lines changed: 74 additions & 50 deletions

File tree

‎patches/sitecustomize.py‎

Lines changed: 74 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,84 @@
11
import os
22

3-
from kaggle_gcp import get_integrations
43
from log import Log
54

65
kaggle_proxy_token = os.getenv("KAGGLE_DATA_PROXY_TOKEN")
76
kernel_integrations_var = os.getenv("KAGGLE_KERNEL_INTEGRATIONS")
87

98
def init():
10-
bq_user_jwt = os.getenv("KAGGLE_USER_SECRETS_TOKEN")
11-
if kaggle_proxy_token or bq_user_jwt:
12-
from google.auth import credentials, environment_vars
13-
from google.cloud import bigquery
14-
from google.cloud.bigquery._http import Connection
15-
# TODO: Update this to the correct kaggle.gcp path once we no longer inject modules
16-
# from the worker.
17-
from kaggle_gcp import PublicBigqueryClient, KaggleKernelCredentials
18-
19-
# If this Kernel has bigquery integration on startup, preload the Kaggle Credentials
20-
# object for magics to work.
21-
if get_integrations().has_bigquery():
22-
from google.cloud.bigquery import magics
23-
magics.context.credentials = KaggleKernelCredentials()
24-
25-
def monkeypatch_bq(bq_client, *args, **kwargs):
26-
specified_credentials = kwargs.get('credentials')
27-
has_bigquery = get_integrations().has_bigquery()
28-
# Prioritize passed in project id, but if it is missing look for env var.
29-
arg_project = kwargs.get('project')
30-
explicit_project_id = arg_project or os.environ.get(environment_vars.PROJECT)
31-
# This is a hack to get around the bug in google-cloud library.
32-
# Remove these two lines once this is resolved:
33-
# https://github.com/googleapis/google-cloud-python/issues/8108
34-
if explicit_project_id:
35-
Log.info(f"Explicit project set to {explicit_project_id}")
36-
kwargs['project'] = explicit_project_id
37-
if explicit_project_id is None and specified_credentials is None and not has_bigquery:
38-
msg = "Using Kaggle's public dataset BigQuery integration."
39-
Log.info(msg)
40-
print(msg)
41-
return PublicBigqueryClient(*args, **kwargs)
42-
43-
else:
44-
if specified_credentials is None:
45-
Log.info("No credentials specified, using KaggleKernelCredentials.")
46-
kwargs['credentials'] = KaggleKernelCredentials()
47-
if (not has_bigquery):
48-
Log.info("No bigquery integration found, creating client anyways.")
49-
print('Please ensure you have selected a BigQuery '
50-
'account in the Kernels Settings sidebar.')
51-
return bq_client(*args, **kwargs)
52-
53-
# Monkey patches BigQuery client creation to use proxy or user-connected GCP account.
54-
# Deprecated in favor of Kaggle.DataProxyClient().
55-
# TODO: Remove this once uses have migrated to that new interface.
56-
bq_client = bigquery.Client
57-
bigquery.Client = lambda *args, **kwargs: monkeypatch_bq(
58-
bq_client, *args, **kwargs)
9+
is_jwe_set = "KAGGLE_USER_SECRETS_TOKEN" in os.environ
10+
if kaggle_proxy_token or is_jwe_set:
11+
init_bigquery()
12+
if is_jwe_set:
13+
if get_integrations().has_gcs():
14+
init_gcs()
15+
16+
17+
def init_bigquery():
18+
from google.auth import credentials, environment_vars
19+
from google.cloud import bigquery
20+
from google.cloud.bigquery._http import Connection
21+
# TODO: Update this to the correct kaggle.gcp path once we no longer inject modules
22+
# from the worker.
23+
from kaggle_gcp import get_integrations, PublicBigqueryClient, KaggleKernelCredentials
24+
25+
# If this Kernel has bigquery integration on startup, preload the Kaggle Credentials
26+
# object for magics to work.
27+
if get_integrations().has_bigquery():
28+
from google.cloud.bigquery import magics
29+
magics.context.credentials = KaggleKernelCredentials()
30+
31+
def monkeypatch_bq(bq_client, *args, **kwargs):
32+
specified_credentials = kwargs.get('credentials')
33+
has_bigquery = get_integrations().has_bigquery()
34+
# Prioritize passed in project id, but if it is missing look for env var.
35+
arg_project = kwargs.get('project')
36+
explicit_project_id = arg_project or os.environ.get(environment_vars.PROJECT)
37+
# This is a hack to get around the bug in google-cloud library.
38+
# Remove these two lines once this is resolved:
39+
# https://github.com/googleapis/google-cloud-python/issues/8108
40+
if explicit_project_id:
41+
Log.info(f"Explicit project set to {explicit_project_id}")
42+
kwargs['project'] = explicit_project_id
43+
if explicit_project_id is None and specified_credentials is None and not has_bigquery:
44+
msg = "Using Kaggle's public dataset BigQuery integration."
45+
Log.info(msg)
46+
print(msg)
47+
return PublicBigqueryClient(*args, **kwargs)
48+
49+
else:
50+
if specified_credentials is None:
51+
Log.info("No credentials specified, using KaggleKernelCredentials.")
52+
kwargs['credentials'] = KaggleKernelCredentials()
53+
if (not has_bigquery):
54+
Log.info("No bigquery integration found, creating client anyways.")
55+
print('Please ensure you have selected a BigQuery '
56+
'account in the Kernels Settings sidebar.')
57+
return bq_client(*args, **kwargs)
58+
59+
# Monkey patches BigQuery client creation to use proxy or user-connected GCP account.
60+
# Deprecated in favor of Kaggle.DataProxyClient().
61+
# TODO: Remove this once uses have migrated to that new interface.
62+
bq_client = bigquery.Client
63+
bigquery.Client = lambda *args, **kwargs: monkeypatch_bq(
64+
bq_client, *args, **kwargs)
65+
66+
67+
def init_gcs():
68+
from kaggle_gcp import get_integrations
69+
if get_integrations().has_gcs():
70+
from kaggle_secrets import GcpTarget
71+
from kaggle_gcp import KaggleKernelCredentials
72+
from google.cloud import storage
73+
def monkeypatch_gcs(gcs_client, *args, **kwargs):
74+
specified_credentials = kwargs.get('credentials')
75+
if specified_credentials is None:
76+
Log.info("No credentials specified, using KaggleKernelCredentials.")
77+
kwargs['credentials'] = KaggleKernelCredentials(target=GcpTarget.GCS)
78+
return gcs_client(*args, **kwargs)
79+
80+
gcs_client = storage.Client
81+
storage.Client = lambda *args, **kwargs: monkeypatch_gcs(gcs_client, *args, **kwargs)
82+
5983

6084
init()

0 commit comments

Comments
 (0)

Back | FazBrowse Home | New Git URL