FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Add message to user when they use private bigquery without project id… by vimota · Pull Request #591 · Kaggle/docker-python · GitHub

/ docker-python Public

Add message to user when they use private bigquery without project id… #591

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vimota merged 2 commits into from
Aug 1, 2019
Conversation 6 Commits 2 Checks 0 Files changed
Merged

Add message to user when they use private bigquery without project id… #591

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
81b9ac0
Add message to user when they use private bigquery without project id…
vimota Jul 24, 2019
e9455f6
Avoid nested monkeypatching.
vimota Aug 1, 2019
File filter

Filter by extension

Filter by extension .py  (3) All 1 file type selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
16 changes: 12 additions & 4 deletions patches/kaggle_gcp.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
import inspect
from google.auth import credentials
from google.auth.exceptions import RefreshError
from google.cloud import bigquery
Expand Down Expand Up @@ -111,6 +112,8 @@ def __init__(self, *args, **kwargs):
# TODO: Remove this once https://github.com/googleapis/google-cloud-python/issues/7122 is implemented.
self._connection = _DataProxyConnection(self)

def has_been_monkeypatched(method):
return "kaggle_gcp" in inspect.getsourcefile(method)

def init_bigquery():
from google.auth import environment_vars
Expand Down Expand Up @@ -145,7 +148,6 @@ def monkeypatch_bq(bq_client, *args, **kwargs):
Log.info(msg)
print(msg)
return PublicBigqueryClient(*args, **kwargs)

else:
if specified_credentials is None:
Log.info("No credentials specified, using KaggleKernelCredentials.")
Expand All @@ -154,14 +156,19 @@ def monkeypatch_bq(bq_client, *args, **kwargs):
Log.info("No bigquery integration found, creating client anyways.")
print('Please ensure you have selected a BigQuery '
'account in the Kernels Settings sidebar.')
if explicit_project_id is None:
Log.info("No project specified while using the unmodified client.")
print('Please ensure you specify a project id when creating the client'
' in order to use your BigQuery account.')
return bq_client(*args, **kwargs)

# Monkey patches BigQuery client creation to use proxy or user-connected GCP account.
# Deprecated in favor of Kaggle.DataProxyClient().
# TODO: Remove this once uses have migrated to that new interface.
bq_client = bigquery.Client
bigquery.Client = lambda *args, **kwargs: monkeypatch_bq(
bq_client, *args, **kwargs)
if (not has_been_monkeypatched(bigquery.Client)):
bigquery.Client = lambda *args, **kwargs: monkeypatch_bq(
bq_client, *args, **kwargs)
return bigquery

def init_gcs():
Expand All @@ -184,7 +191,8 @@ def monkeypatch_gcs(self, *args, **kwargs):
kwargs['credentials'] = KaggleKernelCredentials(target=GcpTarget.GCS)
return gcs_client_init(self, *args, **kwargs)

storage.Client.__init__ = monkeypatch_gcs
if (not has_been_monkeypatched(storage.Client.__init__)):
storage.Client.__init__ = monkeypatch_gcs
return storage

def init():
Expand Down
11 changes: 10 additions & 1 deletion tests/test_bigquery_proxy.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from google.cloud import bigquery
from google.auth.exceptions import DefaultCredentialsError
from google.cloud.bigquery._http import Connection
from kaggle_gcp import KaggleKernelCredentials, PublicBigqueryClient
from kaggle_gcp import KaggleKernelCredentials, PublicBigqueryClient, init_bigquery
import kaggle_secrets


Expand Down Expand Up @@ -67,6 +67,15 @@ def test_proxy_no_project(self):
with env:
client = bigquery.Client()
self._test_proxy(client)

def test_monkeypatching_idempotent(self):
env = EnvironmentVarGuard()
env.unset('KAGGLE_USER_SECRETS_TOKEN')
with env:
client1 = bigquery.Client
init_bigquery()
client2 = bigquery.Client
self.assertEqual(client1, client2)

def test_proxy_with_kwargs(self):
env = EnvironmentVarGuard()
Expand Down
10 changes: 10 additions & 0 deletions tests/test_gcs.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,13 @@ def test_default_credentials_gcs_enabled(self):
init_gcs()
client = storage.Client(project="xyz")
self.assertIsInstance(client._credentials, KaggleKernelCredentials)

def test_monkeypatching_idempotent(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'GCS')
with env:
client1 = storage.Client.__init__
init_gcs()
client2 = storage.Client.__init__
self.assertEqual(client1, client2)
Back | FazBrowse Home | New Git URL