FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Use kaggle-web-client in kaggle_secrets. · feibyte/docker-python@5770a85 · GitHub

/ docker-python Public
forked from Kaggle/docker-python

Commit 5770a85

Browse files
ifigotin
authored
Use kaggle-web-client in kaggle_secrets.
Use kaggle-web-client in kaggle_secrets.
2 parents 953ce15 + 2259a07 commit 5770a85

5 files changed

Lines changed: 29 additions & 77 deletions

File tree

‎patches/kaggle_datasets.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
class KaggleDatasets:
77
GET_GCS_PATH_ENDPOINT = '/requests/CopyDatasetVersionToKnownGcsBucketRequest'
8+
TIMEOUT_SECS = 600
89

910
# Integration types for GCS
1011
AUTO_ML = 1
@@ -20,5 +21,5 @@ def get_gcs_path(self, dataset_dir: str = None) -> str:
2021
'MountSlug': dataset_dir,
2122
'IntegrationType': integration_type,
2223
}
23-
result = self.web_client.make_post_request(data, self.GET_GCS_PATH_ENDPOINT)
24+
result = self.web_client.make_post_request(data, self.GET_GCS_PATH_ENDPOINT, self.TIMEOUT_SECS)
2425
return result['destinationBucket']

‎patches/kaggle_secrets.py‎

Lines changed: 5 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -4,28 +4,12 @@
44
(ie. BigQuery).
55
"""
66

7-
import json
87
import os
9-
import socket
10-
import urllib.request
118
from datetime import datetime, timedelta
129
from enum import Enum, unique
1310
from typing import Optional, Tuple
14-
from urllib.error import HTTPError, URLError
15-
16-
_KAGGLE_DEFAULT_URL_BASE = "https://www.kaggle.com"
17-
_KAGGLE_URL_BASE_ENV_VAR_NAME = "KAGGLE_URL_BASE"
18-
_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME = "KAGGLE_USER_SECRETS_TOKEN"
19-
TIMEOUT_SECS = 40
20-
21-
22-
class CredentialError(Exception):
23-
pass
24-
25-
26-
class BackendError(Exception):
27-
pass
28-
11+
from kaggle_web_client import KaggleWebClient
12+
from kaggle_web_client import (CredentialError, BackendError)
2913

3014
class ValidationError(Exception):
3115
pass
@@ -56,48 +40,9 @@ def service(self):
5640
class UserSecretsClient():
5741
GET_USER_SECRET_ENDPOINT = '/requests/GetUserSecretRequest'
5842
GET_USER_SECRET_BY_LABEL_ENDPOINT = '/requests/GetUserSecretByLabelRequest'
59-
BIGQUERY_TARGET_VALUE = 1
6043

6144
def __init__(self):
62-
url_base_override = os.getenv(_KAGGLE_URL_BASE_ENV_VAR_NAME)
63-
self.url_base = url_base_override or _KAGGLE_DEFAULT_URL_BASE
64-
# Follow the OAuth 2.0 Authorization standard (https://tools.ietf.org/html/rfc6750)
65-
self.jwt_token = os.getenv(_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME)
66-
if self.jwt_token is None:
67-
raise CredentialError(
68-
'A JWT Token is required to use the UserSecretsClient, '
69-
f'but none found in environment variable {_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME}')
70-
self.headers = {'Content-type': 'application/json'}
71-
72-
def _make_post_request(self, data: dict, endpoint: str = GET_USER_SECRET_ENDPOINT) -> dict:
73-
# TODO(b/148309982) This code and the code in the constructor should be
74-
# removed and this class should use the new KaggleWebClient class instead.
75-
url = f'{self.url_base}{endpoint}'
76-
request_body = dict(data)
77-
request_body['JWE'] = self.jwt_token
78-
req = urllib.request.Request(url, headers=self.headers, data=bytes(
79-
json.dumps(request_body), encoding="utf-8"))
80-
try:
81-
with urllib.request.urlopen(req, timeout=TIMEOUT_SECS) as response:
82-
response_json = json.loads(response.read())
83-
if not response_json.get('wasSuccessful') or 'result' not in response_json:
84-
raise BackendError(
85-
f'Unexpected response from the service. Response: {response_json}.')
86-
return response_json['result']
87-
except (URLError, socket.timeout) as e:
88-
if isinstance(
89-
e, socket.timeout) or isinstance(
90-
e.reason, socket.timeout):
91-
raise ConnectionError(
92-
'Timeout error trying to communicate with service. Please ensure internet is on.') from e
93-
raise ConnectionError(
94-
'Connection error trying to communicate with service.') from e
95-
except HTTPError as e:
96-
if e.code == 401 or e.code == 403:
97-
raise CredentialError(
98-
f'Service responded with error code {e.code}.'
99-
' Please ensure you have access to the resource.') from e
100-
raise BackendError('Unexpected response from the service.') from e
45+
self.web_client = KaggleWebClient()
10146

10247
def get_secret(self, label) -> str:
10348
"""Retrieves a user secret value by its label.
@@ -113,7 +58,7 @@ def get_secret(self, label) -> str:
11358
request_body = {
11459
'Label': label,
11560
}
116-
response_json = self._make_post_request(request_body, self.GET_USER_SECRET_BY_LABEL_ENDPOINT)
61+
response_json = self.web_client.make_post_request(request_body, self.GET_USER_SECRET_BY_LABEL_ENDPOINT)
11762
if 'secret' not in response_json:
11863
raise BackendError(
11964
f'Unexpected response from the service. Response: {response_json}')
@@ -174,7 +119,7 @@ def _get_access_token(self, target: GcpTarget) -> Tuple[str, Optional[datetime]]
174119
request_body = {
175120
'Target': target.target
176121
}
177-
response_json = self._make_post_request(request_body)
122+
response_json = self.web_client.make_post_request(request_body, self.GET_USER_SECRET_ENDPOINT)
178123
if 'secret' not in response_json:
179124
raise BackendError(
180125
f'Unexpected response from the service. Response: {response_json}')

‎patches/kaggle_web_client.py‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,22 @@
22
import os
33
import socket
44
import urllib.request
5-
from datetime import datetime, timedelta
6-
from enum import Enum, unique
7-
from typing import Optional, Tuple
85
from urllib.error import HTTPError, URLError
9-
from kaggle_secrets import (_KAGGLE_DEFAULT_URL_BASE,
10-
_KAGGLE_URL_BASE_ENV_VAR_NAME,
11-
_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME,
12-
CredentialError, BackendError, ValidationError)
6+
7+
_KAGGLE_DEFAULT_URL_BASE = "https://www.kaggle.com"
8+
_KAGGLE_URL_BASE_ENV_VAR_NAME = "KAGGLE_URL_BASE"
9+
_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME = "KAGGLE_USER_SECRETS_TOKEN"
10+
TIMEOUT_SECS = 40
11+
12+
class CredentialError(Exception):
13+
pass
14+
15+
16+
class BackendError(Exception):
17+
pass
18+
1319

1420
class KaggleWebClient:
15-
TIMEOUT_SECS = 600
1621

1722
def __init__(self):
1823
url_base_override = os.getenv(_KAGGLE_URL_BASE_ENV_VAR_NAME)
@@ -29,14 +34,14 @@ def __init__(self):
2934
'X-Kaggle-Authorization': f'Bearer {self.jwt_token}',
3035
}
3136

32-
def make_post_request(self, data: dict, endpoint: str) -> dict:
37+
def make_post_request(self, data: dict, endpoint: str, timeout: int = TIMEOUT_SECS) -> dict:
3338
url = f'{self.url_base}{endpoint}'
3439
request_body = dict(data)
3540
request_body['JWE'] = self.jwt_token
3641
req = urllib.request.Request(url, headers=self.headers, data=bytes(
3742
json.dumps(request_body), encoding="utf-8"))
3843
try:
39-
with urllib.request.urlopen(req, timeout=self.TIMEOUT_SECS) as response:
44+
with urllib.request.urlopen(req, timeout=timeout) as response:
4045
response_json = json.loads(response.read())
4146
if not response_json.get('wasSuccessful') or 'result' not in response_json:
4247
raise BackendError(

‎tests/test_datasets.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@
66
from test.support import EnvironmentVarGuard
77
from urllib.parse import urlparse
88

9-
from kaggle_secrets import (_KAGGLE_URL_BASE_ENV_VAR_NAME,
9+
from kaggle_web_client import (KaggleWebClient,
10+
_KAGGLE_URL_BASE_ENV_VAR_NAME,
1011
_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME,
11-
CredentialError, BackendError, ValidationError)
12-
from kaggle_web_client import KaggleWebClient
12+
CredentialError, BackendError)
1313
from kaggle_datasets import KaggleDatasets, _KAGGLE_TPU_NAME_ENV_VAR_NAME
1414

1515
_TEST_JWT = 'test-secrets-key'

‎tests/test_user_secrets.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,11 @@
1010

1111
from google.auth.exceptions import DefaultCredentialsError
1212
from google.cloud import bigquery
13-
from kaggle_secrets import (_KAGGLE_URL_BASE_ENV_VAR_NAME,
13+
from kaggle_secrets import (GcpTarget, UserSecretsClient,
14+
NotFoundError, ValidationError)
15+
from kaggle_web_client import (_KAGGLE_URL_BASE_ENV_VAR_NAME,
1416
_KAGGLE_USER_SECRETS_TOKEN_ENV_VAR_NAME,
15-
CredentialError, GcpTarget, UserSecretsClient,
16-
BackendError, NotFoundError, ValidationError)
17+
CredentialError, BackendError)
1718

1819
_TEST_JWT = 'test-secrets-key'
1920

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL