| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| Expand Up | @@ -209,22 +209,22 @@ class MyMultiModalModel(nn.Module): | |
| self.text_encoder = TextEncoder() | ||
|
|
||
| def forward(self, text_1, text_2, image_1, image_2, audio_1, audio_2): | ||
| emnedding_text_1 = self.text_encoder(text_1) | ||
| emnedding_text_2 = self.text_encoder(text_2) | ||
| embedding_text_1 = self.text_encoder(text_1) | ||
| embedding_text_2 = self.text_encoder(text_2) | ||
|
|
||
| emnedding_image_1 = self.image_encoder(image_1) | ||
| emnedding_image_2 = self.image_encoder(image_2) | ||
| embedding_image_1 = self.image_encoder(image_1) | ||
| embedding_image_2 = self.image_encoder(image_2) | ||
|
|
||
| emnedding_audio_1 = self.image_encoder(audio_1) | ||
| emnedding_audio_2 = self.image_encoder(audio_2) | ||
| embedding_audio_1 = self.image_encoder(audio_1) | ||
| embedding_audio_2 = self.image_encoder(audio_2) | ||
|
|
||
| return ( | ||
| emnedding_text_1, | ||
| emnedding_text_2, | ||
| emnedding_image_1, | ||
| emnedding_image_2, | ||
| emnedding_audio_1, | ||
| emnedding_audio_2, | ||
| embedding_text_1, | ||
| embedding_text_2, | ||
| embedding_image_1, | ||
| embedding_image_2, | ||
| embedding_audio_1, | ||
| embedding_audio_2, | ||
| ) | ||
| ``` | ||
|
|
||
| Expand Down Expand Up | @@ -258,14 +258,14 @@ class MyPodcastModel(nn.Module): | |
| self.image_encoder = ImageEncoder() | ||
| self.text_encoder = TextEncoder() | ||
|
|
||
| def forward_podcast(da: DocumentArray[Podcast]) -> DocumentArray[Podcast]: | ||
| def forward_podcast(self, da: DocumentArray[Podcast]) -> DocumentArray[Podcast]: | ||
| da.audio.embedding = self.audio_encoder(da.audio.tensor) | ||
| da.text.embedding = self.text_encoder(da.text.tensor) | ||
| da.image.embedding = self.image_encoder(da.image.tensor) | ||
|
|
||
| return da | ||
|
|
||
| def forward(da: DocumentArray[PairPodcast]) -> DocumentArray[PairPodcast]: | ||
| def forward(self, da: DocumentArray[PairPodcast]) -> DocumentArray[PairPodcast]: | ||
| da.left = self.forward_podcast(da.left) | ||
| da.right = self.forward_podcast(da.right) | ||
|
|
||
| Expand All | @@ -277,6 +277,49 @@ You instantly win in code readability and maintainability. And for the same pric | |
| schema definition (see below). Everything handles in a pythonic manner by relying on type hints. | ||
|
|
||
|
|
||
| ## Coming from TensorFlow | ||
|
Comment thread
Copy link
Copy Markdown
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Qualitythis part is too big IMO. We just need to show that there is a (tiny ?) difference and that you need to access tensor.tensor. No need to show the full example
Sorry, something went wrong.
All reactions
|
||
|
|
||
| Similar to the PyTorch approach, you can also use DocArray with TensorFlow to handle and represent multi-modal data inside your ML model. | ||
|
|
||
| First off, to use DocArray with TensorFlow we first need to install it as follows: | ||
| ``` | ||
| pip install tensorflow==2.11.0 | ||
| pip install protobuf==3.19.0 | ||
| ``` | ||
|
|
||
| Compared to using DocArray with PyTorch, there is one main difference when using it with TensorFlow:\ | ||
| While DocArray's `TorchTensor` is a subclass of `torch.Tensor`, this is not the case for the `TensorFlowTensor`: Due to technical limitations on `tf.Tensor`, docarray's `TensorFlowTensor` is not a subclass of `tf.Tensor` but instead stores a `tf.Tensor` in its `.tensor` attribute. | ||
|
|
||
| How does this effect you? Whenever you want to access the tensor data to e.g. do operations with it or hand it to your ML model, instead of handing over your `TensorFlowTensor` instance, you need to access its `.tensor` attribute. | ||
|
|
||
| This would look like the following: | ||
|
|
||
| ```python | ||
| from typing import Optional | ||
|
|
||
| from docarray import DocumentArray, BaseDocument | ||
|
|
||
| import tensorflow as tf | ||
|
|
||
|
|
||
| class Podcast(BaseDocument): | ||
| audio_tensor: Optional[AudioTensorFlowTensor] | ||
| embedding: Optional[AudioTensorFlowTensor] | ||
|
|
||
|
|
||
| class MyPodcastModel(tf.keras.Model): | ||
| def __init__(self): | ||
| super().__init__() | ||
| self.audio_encoder = AudioEncoder() | ||
|
|
||
| def call(self, inputs: DocumentArray[Podcast]) -> DocumentArray[Podcast]: | ||
| inputs.audio_tensor.embedding = self.audio_encoder( | ||
| inputs.audio_tensor.tensor | ||
| ) # access audio_tensor's .tensor attribute | ||
| return inputs | ||
| ``` | ||
|
|
||
|
|
||
|
|
||
| ## Coming from FastAPI | ||
|
|
||
| Expand Down | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| Expand Up | @@ -27,14 +27,22 @@ | |
| from pydantic.fields import ModelField | ||
|
|
||
| from docarray.proto import DocumentArrayStackedProto | ||
| from docarray.typing import TorchTensor | ||
| from docarray.typing.tensor.abstract_tensor import AbstractTensor | ||
|
|
||
| try: | ||
| from docarray.typing import TorchTensor | ||
| except ImportError: | ||
| TorchTensor = None # type: ignore | ||
|
|
||
| try: | ||
| import tensorflow as tf # type: ignore | ||
|
|
||
| from docarray.typing import TensorFlowTensor | ||
|
Comment thread
Comment on lines
+36
to
+39
Copy link
Copy Markdown
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Qualityfor torch i moved this thing to a helper in utils, so this check only has to be done once globally. Can we do the same for tf?
Sorry, something went wrong.
All reactions
Copy link
Copy Markdown
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Qualityyes, saw that and started doing this in the TF embedding/video/audio PR, so i'll do this refactor there if that's fine with you @JohannesMessner
Sorry, something went wrong.
samsja reacted with thumbs up emoji
All reactions
|
||
|
|
||
| tf_available = True | ||
| except (ImportError, TypeError): | ||
| TensorFlowTensor = None # type: ignore | ||
| tf_available = False | ||
|
|
||
| T = TypeVar('T', bound='DocumentArrayStacked') | ||
| IndexIterType = Union[slice, Iterable[int], Iterable[bool], None] | ||
|
|
||
| Expand Down Expand Up | @@ -163,7 +171,26 @@ def _create_columns( | |
| tensor_columns: Dict[str, AbstractTensor] = dict() | ||
|
|
||
| for field, type_ in column_schema.items(): | ||
| if issubclass(type_, AbstractTensor): | ||
| if tf_available and isinstance(getattr(docs[0], field), TensorFlowTensor): | ||
| # tf.Tensor does not allow item assignment, therefore the optimized way | ||
| # of initializing an empty array and assigning values to it iteratively | ||
| # does not work here, therefore handle separately. | ||
| tf_stack = [] | ||
|
Comment thread
anna-charlotte marked this conversation as resolved.
|
||
| for i, doc in enumerate(docs): | ||
| val = getattr(doc, field) | ||
| if val is None: | ||
| val = tensor_type.get_comp_backend().none_value() | ||
| tf_stack.append(val.tensor) | ||
| del val.tensor | ||
|
|
||
| stacked: tf.Tensor = tf.stack(tf_stack) | ||
| tensor_columns[field] = TensorFlowTensor(stacked) | ||
| for i, doc in enumerate(docs): | ||
| val = getattr(doc, field) | ||
| x = tensor_columns[field][i].tensor | ||
| val.tensor = x | ||
|
|
||
| elif issubclass(type_, AbstractTensor): | ||
| tensor = getattr(docs[0], field) | ||
| column_shape = ( | ||
| (len(docs), *tensor.shape) if tensor is not None else (len(docs),) | ||
| Expand All | @@ -190,7 +217,8 @@ def _create_columns( | |
| # We thus chose to convert the individual rank 0 tensors to rank 1 | ||
| # This does mean that stacking rank 0 tensors will transform them | ||
| # to rank 1 | ||
| if tensor_columns[field].ndim == 1: | ||
| tensor = tensor_columns[field] | ||
| if tensor.get_comp_backend().n_dim(tensor) == 1: | ||
| setattr(doc, field, tensor_columns[field][i : i + 1]) | ||
| else: | ||
| setattr(doc, field, tensor_columns[field][i]) | ||
| Expand Down | ||
| Back | FazBrowse Home | New Git URL |
Uh oh!
There was an error while loading. Please reload this page.