Commit 4bd7c38
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,22 +1,56 @@ | |||
| 1 | 1 | # TODO(rosbo): Remove this once we fix the issue with fastai importing older libcudnn if imported prior to tensorflow | |
| 2 | 2 | import tensorflow | |
| 3 | 3 | import os | |
| 4 | + from google.auth import credentials | ||
| 5 | + from google.cloud import bigquery | ||
| 6 | + from google.cloud.bigquery._http import Connection | ||
| 4 | 7 | ||
| 5 | - # Monkey patches BigQuery client creation to use proxy. | ||
| 6 | - kaggle_proxy_token = os.getenv("KAGGLE_DATA_PROXY_TOKEN") | ||
| 7 | - if kaggle_proxy_token: | ||
| 8 | - from google.auth import credentials | ||
| 9 | - from google.cloud import bigquery | ||
| 10 | - from google.cloud.bigquery._http import Connection | ||
| 8 | + class KaggleKernelCredentials(credentials.Credentials): | ||
| 9 | + def refresh(self, request): | ||
| 10 | + print("Calling Kaggle.UserSecrets to refresh token.") | ||
| 11 | + # Set self.token and self.expiry here. | ||
| 12 | + raise NotImplementedError("Private BigQuery integration is not yet implemented.") | ||
| 11 | 13 | ||
| 12 | - Connection.API_BASE_URL = os.getenv("KAGGLE_DATA_PROXY_URL") | ||
| 13 | - Connection._EXTRA_HEADERS["X-KAGGLE-PROXY-DATA"] = kaggle_proxy_token | ||
| 14 | + kaggle_proxy_token = os.getenv("KAGGLE_DATA_PROXY_TOKEN") | ||
| 15 | + CONNECTION_BASE_URL = Connection.API_BASE_URL | ||
| 16 | + def monkeypatch_bq(bq_client, *args, **kwargs): | ||
| 17 | + data_proxy_project = os.getenv("KAGGLE_DATA_PROXY_PROJECT") | ||
| 18 | + bq_user_jwt = os.getenv("KAGGLE_BQ_USER_JWT") | ||
| 19 | + specified_project = kwargs.get('project').lower() if 'project' in kwargs else None | ||
| 20 | + # Use Data Proxy if user has specified to use the Kaggle project, or if | ||
| 21 | + # there are no connected GCP accounts (to maintain backwards compatibility). | ||
| 22 | + if bq_user_jwt is None and specified_project and specified_project != 'kaggle': | ||
| 23 | + raise Exception("In order to query a private BigQuery project, please connect a GCP account. " | ||
| 24 | + "Otherwise specify 'kaggle' as the project to use Kaggle's public dataset BigQuery integration.") | ||
| 25 | + use_data_proxy = specified_project == 'kaggle' or bq_user_jwt is None | ||
| 26 | + if use_data_proxy: | ||
| 27 | + if data_proxy_project is None or kaggle_proxy_token is None: | ||
| 28 | + # We don't have the data proxy info so leave the bq client unmodified. | ||
| 29 | + return bq_client(*args, **kwargs) | ||
| 30 | + print("Using Kaggle's public dataset BigQuery integration.") | ||
| 31 | + Connection.API_BASE_URL = os.getenv("KAGGLE_DATA_PROXY_URL") | ||
| 32 | + Connection._EXTRA_HEADERS["X-KAGGLE-PROXY-DATA"] = kaggle_proxy_token | ||
| 33 | + anon_credentials = credentials.AnonymousCredentials() | ||
| 34 | + anon_credentials.refresh = lambda *args: None | ||
| 35 | + kwargs['project'] = data_proxy_project | ||
| 36 | + return bq_client( | ||
| 37 | + *args, | ||
| 38 | + credentials=anon_credentials, | ||
| 39 | + **kwargs) | ||
| 40 | + else: | ||
| 41 | + Connection.API_BASE_URL = CONNECTION_BASE_URL | ||
| 42 | + Connection._EXTRA_HEADERS.pop('X-KAGGLE-PROXY-DATA', None) | ||
| 43 | + if kwargs.get('credentials') is not None: | ||
| 44 | + # The user wants to use their own credentials scheme, don't try to interfere. | ||
| 45 | + return bq_client(*args, **kwargs) | ||
| 46 | + print("Using connected BigQuery Account.") | ||
| 47 | + kwargs['credentials'] = KaggleKernelCredentials() | ||
| 48 | + kwargs['project'] = kwargs.get('project') | ||
| 49 | + return bq_client( | ||
| 50 | + *args, | ||
| 51 | + **kwargs) | ||
| 14 | 52 | ||
| 15 | - bq_client = bigquery.Client | ||
| 16 | - bigquery.Client = lambda *args, **kwargs: bq_client( | ||
| 17 | - *args, | ||
| 18 | - credentials=credentials.AnonymousCredentials(), | ||
| 19 | - project=os.getenv("KAGGLE_DATA_PROXY_PROJECT"), | ||
| 20 | - **kwargs) | ||
| 53 | + # Monkey patches BigQuery client creation to use proxy or user. | ||
| 54 | + bq_client = bigquery.Client | ||
| 55 | + bigquery.Client = lambda *args, **kwargs: monkeypatch_bq(bq_client, *args, **kwargs) | ||
| 21 | 56 | ||
| 22 | - credentials.AnonymousCredentials.refresh = lambda *args: None | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -11,19 +11,26 @@ | |||
| 11 | 11 | URL = "http://%s:%s" % (HOSTNAME, PORT) | |
| 12 | 12 | ||
| 13 | 13 | class TestBigQuery(unittest.TestCase): | |
| 14 | - def test_proxy(self): | ||
| 15 | - httpd = HTTPServer((HOSTNAME, PORT), HTTPHandler) | ||
| 16 | - threading.Thread(target=httpd.serve_forever).start() | ||
| 14 | + def _test_proxy(self, client): | ||
| 15 | + with HTTPServer((HOSTNAME, PORT), HTTPHandler) as httpd: | ||
| 16 | + threading.Thread(target=httpd.serve_forever).start() | ||
| 17 | + | ||
| 18 | + try: | ||
| 19 | + for ds in client.list_datasets(): pass | ||
| 20 | + except: | ||
| 21 | + pass | ||
| 22 | + | ||
| 23 | + httpd.shutdown() | ||
| 24 | + self.assertTrue(HTTPHandler.called, msg="Fake server did not recieve a request from the BQ client.") | ||
| 25 | + self.assertTrue(HTTPHandler.header_found, msg="X-KAGGLE-PROXY-DATA header was missing from the BQ request.") | ||
| 26 | + | ||
| 27 | + def test_proxy_kaggle_project(self): | ||
| 28 | + client = bigquery.Client(project='KAGGLE') | ||
| 29 | + self._test_proxy(client) | ||
| 30 | + | ||
| 31 | + def test_proxy_no_project(self): | ||
| 17 | 32 | client = bigquery.Client() | |
| 18 | - | ||
| 19 | - try: | ||
| 20 | - for ds in client.list_datasets(): pass | ||
| 21 | - except: | ||
| 22 | - pass | ||
| 23 | - | ||
| 24 | - httpd.shutdown() | ||
| 25 | - self.assertTrue(HTTPHandler.called, msg="Fake server did not recieve a request from the BQ client.") | ||
| 26 | - self.assertTrue(HTTPHandler.header_found, msg="X-KAGGLE-PROXY-DATA header was missing from the BQ request.") | ||
| 33 | + self._test_proxy(client) | ||
| 27 | 34 | ||
| 28 | 35 | class HTTPHandler(BaseHTTPRequestHandler): | |
| 29 | 36 | called = False | |
0 commit comments