FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Simplified Cloud AI API integration. by kornelregius · Pull Request #932 · Kaggle/docker-python · GitHub

/ docker-python Public

Simplified Cloud AI API integration. #932

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rosbo merged 9 commits into from
Jan 13, 2021
Conversation 6 Commits 9 Checks 0 Files changed
Merged

Simplified Cloud AI API integration. #932

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
30ff635
Cloud Translation, Natural language, Vision,
kornelregius Dec 20, 2020
ef87f07
Fixes for style guide
kornelregius Dec 22, 2020
3fe8ec9
Merge https://github.com/Kaggle/docker-python
kornelregius Jan 5, 2021
57c0e7c
Merge https://github.com/Kaggle/docker-python
kornelregius Jan 11, 2021
6f9a5d6
Simplified Cloud AI Platform integration
kornelregius Jan 11, 2021
4d9f7f9
Simplified Cloud AI integration
kornelregius Jan 11, 2021
8b98fea
Merge branch 'master' of https://github.com/kornelregius/docker-python
kornelregius Jan 11, 2021
8bdc39f
Fix merge issue in dev.Dockerfile
kornelregius Jan 12, 2021
88291b1
Merge https://github.com/Kaggle/docker-python
kornelregius Jan 12, 2021
File filter

Filter by extension

Filter by extension .Dockerfile  (1) .py  (9) All 2 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
3 changes: 2 additions & 1 deletion dev.Dockerfile
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,5 @@ ADD patches/kaggle_gcp.py /root/.local/lib/python3.7/site-packages/kaggle_gcp.py
ADD patches/kaggle_secrets.py /root/.local/lib/python3.7/site-packages/kaggle_secrets.py
ADD patches/kaggle_session.py /root/.local/lib/python3.7/site-packages/kaggle_session.py
ADD patches/kaggle_web_client.py /root/.local/lib/python3.7/site-packages/kaggle_web_client.py
ADD patches/kaggle_datasets.py /root/.local/lib/python3.7/site-packages/kaggle_datasets.py
ADD patches/kaggle_datasets.py /root/.local/lib/python3.7/site-packages/kaggle_datasets.py
ADD patches/sitecustomize.py /root/.local/lib/python3.7/site-packages/sitecustomize.py
53 changes: 17 additions & 36 deletions patches/kaggle_gcp.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -42,20 +42,9 @@ def has_bigquery(self):
def has_gcs(self):
return GcpTarget.GCS in self.integrations

def has_automl(self):
return GcpTarget.AUTOML in self.integrations

def has_translation(self):
return GcpTarget.TRANSLATION in self.integrations

def has_natural_language(self):
return GcpTarget.NATURAL_LANGUAGE in self.integrations

def has_video_intelligence(self):
return GcpTarget.VIDEO_INTELLIGENCE in self.integrations

def has_vision(self):
return GcpTarget.VISION in self.integrations
def has_cloudai(self):
return GcpTarget.CLOUDAI in self.integrations or \
GcpTarget.AUTOML in self.integrations

class KaggleKernelCredentials(credentials.Credentials):
"""Custom Credentials used to authenticate using the Kernel's connected OAuth account.
Expand All @@ -74,16 +63,8 @@ def refresh(self, request):
self.token, self.expiry = client.get_bigquery_access_token()
elif self.target == GcpTarget.GCS:
self.token, self.expiry = client._get_gcs_access_token()
elif self.target == GcpTarget.AUTOML:
self.token, self.expiry = client._get_automl_access_token()
elif self.target == GcpTarget.TRANSLATION:
self.token, self.expiry = client._get_translation_access_token()
elif self.target == GcpTarget.NATURAL_LANGUAGE:
self.token, self.expiry = client._get_natural_language_access_token()
elif self.target == GcpTarget.VIDEO_INTELLIGENCE:
self.token, self.expiry = client._get_video_intelligence_access_token()
elif self.target == GcpTarget.VISION:
self.token, self.expiry = client._get_vision_access_token()
elif self.target == GcpTarget.CLOUDAI:
self.token, self.expiry = client._get_cloudai_access_token()
except ConnectionError as e:
Log.error(f"Connection error trying to refresh access token: {e}")
print("There was a connection error trying to fetch the access token. "
Expand Down Expand Up @@ -262,12 +243,12 @@ def init_automl():
return

from kaggle_gcp import get_integrations
if not get_integrations().has_automl():
if not get_integrations().has_cloudai():
return

from kaggle_secrets import GcpTarget
from kaggle_gcp import KaggleKernelCredentials
kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.AUTOML)
kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)

# Patch the 2 GA clients: AutoMlClient and PreditionServiceClient
monkeypatch_client(automl.AutoMlClient, kaggle_kernel_credentials)
Expand All @@ -293,10 +274,10 @@ def init_translation_v2():
return translate_v2

from kaggle_gcp import get_integrations
if not get_integrations().has_translation():
if not get_integrations().has_cloudai():
return translate_v2
from kaggle_secrets import GcpTarget
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION)
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
monkeypatch_client(translate_v2.Client, kernel_credentials)
return translate_v2

Expand All @@ -307,10 +288,10 @@ def init_translation_v3():
return translate_v3

from kaggle_gcp import get_integrations
if not get_integrations().has_translation():
if not get_integrations().has_cloudai():
return translate_v3
from kaggle_secrets import GcpTarget
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION)
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
monkeypatch_client(translate_v3.TranslationServiceClient, kernel_credentials)
return translate_v3

Expand All @@ -320,11 +301,11 @@ def init_natural_language():
return language

from kaggle_gcp import get_integrations
if not get_integrations().has_natural_language():
if not get_integrations().has_cloudai():
return language

from kaggle_secrets import GcpTarget
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.NATURAL_LANGUAGE)
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
monkeypatch_client(language.LanguageServiceClient, kernel_credentials)
monkeypatch_client(language.LanguageServiceAsyncClient, kernel_credentials)
return language
Expand All @@ -335,11 +316,11 @@ def init_video_intelligence():
return videointelligence

from kaggle_gcp import get_integrations
if not get_integrations().has_video_intelligence():
if not get_integrations().has_cloudai():
return videointelligence

from kaggle_secrets import GcpTarget
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.VIDEO_INTELLIGENCE)
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
monkeypatch_client(
videointelligence.VideoIntelligenceServiceClient,
kernel_credentials)
Expand All @@ -354,11 +335,11 @@ def init_vision():
return vision

from kaggle_gcp import get_integrations
if not get_integrations().has_vision():
if not get_integrations().has_cloudai():
return vision

from kaggle_secrets import GcpTarget
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.VISION)
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
monkeypatch_client(vision.ImageAnnotatorClient, kernel_credentials)
monkeypatch_client(vision.ImageAnnotatorAsyncClient, kernel_credentials)
return vision
Expand Down
22 changes: 4 additions & 18 deletions patches/kaggle_secrets.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,9 @@ class GcpTarget(Enum):
"""Enum class to store GCP targets."""
BIGQUERY = (1, "BigQuery")
GCS = (2, "Google Cloud Storage")
# Old name, should remove later.
AUTOML = (3, "Cloud AutoML")
TRANSLATION = (4, "Cloud Translation")
NATURAL_LANGUAGE = (5, "Cloud Natural Language")
VIDEO_INTELLIGENCE = (6, "Cloud Video Intelligence")
VISION = (7, "Cloud Vision")
CLOUDAI = (3, "Google Cloud AI Platform")

def __init__(self, target, service):
self._target = target
Expand Down Expand Up @@ -155,20 +153,8 @@ def _write_gsutil_credentials_file(self, credentials) -> str:
def _get_gcs_access_token(self) -> Tuple[str, Optional[datetime]]:
return self._get_access_token(GcpTarget.GCS)

def _get_automl_access_token(self) -> Tuple[str, Optional[datetime]]:
return self._get_access_token(GcpTarget.AUTOML)

def _get_translation_access_token(self) -> Tuple[str, Optional[datetime]]:
return self._get_access_token(GcpTarget.TRANSLATION)

def _get_natural_language_access_token(self) -> Tuple[str, Optional[datetime]]:
return self._get_access_token(GcpTarget.NATURAL_LANGUAGE)

def _get_video_intelligence_access_token(self) -> Tuple[str, Optional[datetime]]:
return self._get_access_token(GcpTarget.VIDEO_INTELLIGENCE)

def _get_vision_access_token(self) -> Tuple[str, Optional[datetime]]:
return self._get_access_token(GcpTarget.VISION)
def _get_cloudai_access_token(self) -> Tuple[str, Optional[datetime]]:
return self._get_access_token(GcpTarget.CLOUDAI)

def _get_access_token(self, target: GcpTarget) -> Tuple[str, Optional[datetime]]:
request_body = {
Expand Down
30 changes: 22 additions & 8 deletions tests/test_automl.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ def test_user_provided_credentials(self):
credentials = _make_credentials()
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_automl()
client = automl.AutoMlClient(credentials=credentials)
Expand All @@ -48,7 +48,7 @@ def test_tables_client_credentials(self):
credentials = _make_credentials()
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_automl()
tables_client = automl_v1beta1.TablesClient(credentials=credentials)
Expand All @@ -58,7 +58,7 @@ def test_tables_client_credentials(self):
def test_default_credentials_automl_client(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_automl()
automl_client = automl.AutoMlClient()
Expand All @@ -70,7 +70,7 @@ def test_default_credentials_automl_client(self):
def test_default_credentials_automl_v1beta1_client(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_automl()
automl_client = automl_v1beta1.AutoMlClient()
Expand All @@ -82,7 +82,7 @@ def test_default_credentials_automl_v1beta1_client(self):
def test_default_credentials_tables_client(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_automl()
tables_client = automl_v1beta1.TablesClient()
Expand All @@ -94,7 +94,7 @@ def test_default_credentials_tables_client(self):
def test_default_credentials_prediction_client(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
prediction_client = automl.PredictionServiceClient()
self.assertIsNotNone(prediction_client.credentials)
Expand All @@ -105,7 +105,7 @@ def test_default_credentials_prediction_client(self):
def test_default_credentials_prediction_v1beta1_client(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
prediction_client = automl_v1beta1.PredictionServiceClient()
self.assertIsNotNone(prediction_client.credentials)
Expand All @@ -115,9 +115,23 @@ def test_default_credentials_prediction_v1beta1_client(self):
def test_monkeypatching_idempotent(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
client1 = automl.AutoMlClient.__init__
init_automl()
client2 = automl.AutoMlClient.__init__
self.assertEqual(client1, client2)

@patch("google.cloud.automl_v1beta1.PredictionServiceClient", new=FakeClient)
def test_legacy_AUTOML_variable_v1beta1_client(self):
"""
Tests previous KAGGLE_KERNEL_INTEGRATIONS="AUTOML" environment setting
"""
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
with env:
prediction_client = automl_v1beta1.PredictionServiceClient()
self.assertIsNotNone(prediction_client.credentials)
self.assertIsInstance(prediction_client.credentials, KaggleKernelCredentials)
self.assertTrue(prediction_client._connection.user_agent.startswith("kaggle-gcp-client/1.0"))
8 changes: 4 additions & 4 deletions tests/test_natural_language.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def __init__(self, user_agent):
def test_default_credentials(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_natural_language()
client = language.LanguageServiceClient()
Expand All @@ -38,7 +38,7 @@ def test_user_provided_credentials(self):
credentials = _make_credentials()
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_natural_language()
client = language.LanguageServiceClient(credentials=credentials)
Expand All @@ -49,7 +49,7 @@ def test_user_provided_credentials(self):
def test_monkeypatching_succeed(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_natural_language()
client = language.LanguageServiceClient.__init__
Expand All @@ -58,7 +58,7 @@ def test_monkeypatching_succeed(self):
def test_monkeypatching_idempotent(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_natural_language()
client1 = language.LanguageServiceClient.__init__
Expand Down
4 changes: 2 additions & 2 deletions tests/test_tensorflow_credentials.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ def test_set_tensorflow_credential(self, mock_configure_gcs):
credential = '{"client_id":"fake_client_id",' \
'"client_secret":"fake_client_secret",' \
'"refresh_token":"not a refresh token",' \
'"type":"authorized_user"}';
'"type":"authorized_user"}'

env = EnvironmentVarGuard()
env.set('HOME', '/tmp')
Expand All @@ -22,7 +22,7 @@ def test_set_tensorflow_credential(self, mock_configure_gcs):
# These need to be set to make UserSecretsClient happy, but aren't
# pertinent to this test.
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')

user_secrets = UserSecretsClient()
user_secrets.set_tensorflow_credential(credential)
Expand Down
14 changes: 7 additions & 7 deletions tests/test_translation.py
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def __init__(self, user_agent):
def test_default_credentials_v2(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_translation_v2()
client = translate_v2.Client()
Expand All @@ -41,7 +41,7 @@ def test_user_provided_credentials_v2(self):
credentials = _make_credentials()
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_translation_v2()
client = translate_v2.Client(credentials=credentials)
Expand All @@ -52,7 +52,7 @@ def test_user_provided_credentials_v2(self):
def test_default_credentials_v3(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_translation_v3()
client = translate.TranslationServiceClient()
Expand All @@ -65,7 +65,7 @@ def test_user_provided_credentials_v3(self):
credentials = _make_credentials()
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_translation_v3()
client = translate.TranslationServiceClient(credentials=credentials)
Expand All @@ -76,7 +76,7 @@ def test_user_provided_credentials_v3(self):
def test_monkeypatching_succeed(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_translation_v2()
init_translation_v3()
Expand All @@ -90,7 +90,7 @@ def test_monkeypatching_succeed(self):
def test_monkeypatching_idempotent(self):
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_translation_v2()
init_translation_v3()
Expand All @@ -116,7 +116,7 @@ def test_client_credential_uniqueness_v3(self):
credentials = _make_credentials()
env = EnvironmentVarGuard()
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
with env:
init_translation_v3()
client1 = translate.TranslationServiceClient()
Expand Down
Loading
Back | FazBrowse Home | New Git URL