FazBrowse GitHub Viewer
|
Trending
|
URL:
|
Home
Tools:
[Download Repo ZIP]
[View Raw Code]
[Original HTTPS Page]
docarray/docarray/computation/abstract_comp_backend.py at main · docarray/docarray · GitHub
Uh oh!
There was an error while loading.
Please reload this page
.
docarray
/
docarray
Public
Notifications
You must be signed in to change notification settings
Fork
243
Star
3.1k
Code
Issues
68
Pull requests
43
Discussions
Actions
Security and quality
0
Insights
Additional navigation options
Code
Issues
Pull requests
Discussions
Actions
Security and quality
Insights
Expand file tree
Breadcrumbs
docarray
/
docarray
/
computation
/
abstract_comp_backend.py
Copy path
More file actions
More file actions
Latest commit
History
History
History
278 lines (242 loc) · 9.48 KB
Breadcrumbs
docarray
/
docarray
/
computation
/
abstract_comp_backend.py
Copy path
File metadata and controls
278 lines (242 loc) · 9.48 KB
Raw
Copy raw file
Download raw file
Open symbols panel
Edit and raw actions
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
import
typing
from
abc
import
ABC
,
abstractmethod
from
typing
import
TYPE_CHECKING
,
Any
,
List
,
Optional
,
Tuple
,
TypeVar
,
Union
,
Iterable
if
TYPE_CHECKING
:
import
numpy
as
np
# In practice all of the below will be the same type
TTensor
=
TypeVar
(
'TTensor'
)
TTensorRetrieval
=
TypeVar
(
'TTensorRetrieval'
,
bound
=
Iterable
)
TTensorMetrics
=
TypeVar
(
'TTensorMetrics'
)
class
AbstractComputationalBackend
(
ABC
,
typing
.
Generic
[
TTensor
]):
"""
Abstract base class for computational backends.
Every supported tensor/ML framework (numpy, torch etc.) should define its own
computational backend exposing common functionality expressed in that framework.
That way, DocList can leverage native implementations from all frameworks.
"""
@
classmethod
@
abstractmethod
def
stack
(
cls
,
tensors
:
Union
[
List
[
'TTensor'
],
Tuple
[
'TTensor'
]],
dim
:
int
=
0
)
->
'TTensor'
:
"""
Stack a list of tensors along a new axis.
"""
...
@
classmethod
@
abstractmethod
def
copy
(
cls
,
tensor
:
'TTensor'
)
->
'TTensor'
:
"""return a copy of the tensor"""
...
@
classmethod
@
abstractmethod
def
n_dim
(
cls
,
array
:
'TTensor'
)
->
int
:
"""
Get the number of the array dimensions.
"""
...
@
classmethod
@
abstractmethod
def
squeeze
(
cls
,
tensor
:
'TTensor'
)
->
'TTensor'
:
"""
Returns a tensor with all the dimensions of tensor of size 1 removed.
"""
...
@
classmethod
@
abstractmethod
def
to_numpy
(
cls
,
array
:
'TTensor'
)
->
'np.ndarray'
:
"""
Convert array to np.ndarray.
"""
...
@
classmethod
@
abstractmethod
def
empty
(
cls
,
shape
:
Tuple
[
int
, ...],
dtype
:
Optional
[
Any
]
=
None
,
device
:
Optional
[
Any
]
=
None
,
)
->
'TTensor'
:
...
@
classmethod
@
abstractmethod
def
none_value
(
cls
)
->
typing
.
Any
:
"""Provide a compatible value that represents None in the Tensor Backend."""
...
@
classmethod
@
abstractmethod
def
to_device
(
cls
,
tensor
:
'TTensor'
,
device
:
str
)
->
'TTensor'
:
"""Move the tensor to the specified device."""
...
@
classmethod
@
abstractmethod
def
device
(
cls
,
tensor
:
'TTensor'
)
->
Optional
[
str
]:
"""Return device on which the tensor is allocated."""
...
@
classmethod
@
abstractmethod
def
shape
(
cls
,
tensor
:
'TTensor'
)
->
Tuple
[
int
, ...]:
"""Get shape of tensor"""
...
@
classmethod
@
abstractmethod
def
reshape
(
cls
,
tensor
:
'TTensor'
,
shape
:
Tuple
[
int
, ...])
->
'TTensor'
:
"""
Gives a new shape to tensor without changing its data.
:param tensor: tensor to be reshaped
:param shape: the new shape
:return: a tensor with the same data and number of elements as tensor
but with the specified shape.
"""
...
@
classmethod
@
abstractmethod
def
detach
(
cls
,
tensor
:
'TTensor'
)
->
'TTensor'
:
"""
Returns the tensor detached from its current graph.
:param tensor: tensor to be detached
:return: a detached tensor with the same data.
"""
...
@
classmethod
@
abstractmethod
def
dtype
(
cls
,
tensor
:
'TTensor'
)
->
Any
:
"""Get the data type of the tensor."""
...
@
classmethod
@
abstractmethod
def
isnan
(
cls
,
tensor
:
'TTensor'
)
->
'TTensor'
:
"""Check element-wise for nan and return result as a boolean array"""
...
@
classmethod
@
abstractmethod
def
minmax_normalize
(
cls
,
tensor
:
'TTensor'
,
t_range
:
Tuple
=
(
0
,
1
),
x_range
:
Optional
[
Tuple
]
=
None
,
eps
:
float
=
1e-7
,
)
->
'TTensor'
:
"""
Normalize values in `tensor` into `t_range`.
`tensor` can be a 1D array or a 2D array. When `tensor` is a 2D array, then
normalization is row-based.
!!! note
- with `t_range=(0, 1)` will normalize the min-value of data to 0, max to 1;
- with `t_range=(1, 0)` will normalize the min-value of data to 1, max value
of the data to 0.
:param tensor: the data to be normalized
:param t_range: a tuple represents the target range.
:param x_range: a tuple represents tensors range.
:param eps: a small jitter to avoid divide by zero
:return: normalized data in `t_range`
"""
...
@
classmethod
@
abstractmethod
def
equal
(
cls
,
tensor1
:
'TTensor'
,
tensor2
:
'TTensor'
)
->
bool
:
"""
Check if two tensors are equal.
:param tensor1: the first tensor
:param tensor2: the second tensor
:return: True if two tensors are equal, False otherwise.
If one or more of the inputs is not a tensor of this framework, return False.
"""
...
class
Retrieval
(
ABC
,
typing
.
Generic
[
TTensorRetrieval
]):
"""
Abstract class for retrieval and ranking functionalities
"""
@
staticmethod
@
abstractmethod
def
top_k
(
values
:
'TTensorRetrieval'
,
k
:
int
,
descending
:
bool
=
False
,
device
:
Optional
[
str
]
=
None
,
)
->
Tuple
[
'TTensorRetrieval'
,
'TTensorRetrieval'
]:
"""
Retrieves the top k smallest values in `values`,
and returns them alongside their indices in the input `values`.
Can also be used to retrieve the top k largest values,
by setting the `descending` flag to True.
:param values: Tensor of values to rank.
Should be of shape (n_queries, n_values_per_query).
Inputs of shape (n_values_per_query,) will be expanded
to (1, n_values_per_query).
:param k: number of values to retrieve
:param descending: retrieve largest values instead of smallest values
:param device: the computational device to use.
:return: Tuple containing the retrieved values, and their indices.
Both ar of shape (n_queries, k)
"""
...
class
Metrics
(
ABC
,
typing
.
Generic
[
TTensorMetrics
]):
"""
Abstract base class for metrics (distances and similarities).
"""
@
staticmethod
@
abstractmethod
def
cosine_sim
(
x_mat
:
'TTensorMetrics'
,
y_mat
:
'TTensorMetrics'
,
eps
:
float
=
1e-7
,
device
:
Optional
[
str
]
=
None
,
)
->
'TTensorMetrics'
:
"""Pairwise cosine similarities between all vectors in x_mat and y_mat.
:param x_mat: tensor of shape (n_vectors, n_dim), where n_vectors is the
number of vectors and n_dim is the number of dimensions of each example.
:param y_mat: tensor of shape (n_vectors, n_dim), where n_vectors is the
number of vectors and n_dim is the number of dimensions of each example.
:param eps: a small jitter to avoid divde by zero
:param device: the device to use for computations.
If not provided, the devices of x_mat and y_mat are used.
:return: Tensor of shape (n_vectors, n_vectors) containing all pairwise
cosine distances.
The index [i_x, i_y] contains the cosine distance between
x_mat[i_x] and y_mat[i_y].
"""
...
@
staticmethod
@
abstractmethod
def
euclidean_dist
(
x_mat
:
'TTensorMetrics'
,
y_mat
:
'TTensorMetrics'
,
device
:
Optional
[
str
]
=
None
,
)
->
'TTensorMetrics'
:
"""Pairwise Euclidian distances between all vectors in x_mat and y_mat.
:param x_mat: tensor of shape (n_vectors, n_dim), where n_vectors is the
number of vectors and n_dim is the number of dimensions of each example.
:param y_mat: tensor of shape (n_vectors, n_dim), where n_vectors is the
number of vectors and n_dim is the number of dimensions of each example.
:param device: the device to use for pytorch computations.
If not provided, the devices of x_mat and y_mat are used.
:return: Tensor of shape (n_vectors, n_vectors) containing all pairwise
euclidian distances.
The index [i_x, i_y] contains the euclidian distance between
x_mat[i_x] and y_mat[i_y].
"""
...
@
staticmethod
@
abstractmethod
def
sqeuclidean_dist
(
x_mat
:
'TTensorMetrics'
,
y_mat
:
'TTensorMetrics'
,
device
:
Optional
[
str
]
=
None
,
)
->
'TTensorMetrics'
:
"""Pairwise Squared Euclidian distances between all vectors
in x_mat and y_mat.
:param x_mat: tensor of shape (n_vectors, n_dim), where n_vectors is the
number of vectors and n_dim is the number of dimensions of each
example.
:param y_mat: tensor of shape (n_vectors, n_dim), where n_vectors is the
number of vectors and n_dim is the number of dimensions of each
example.
:param device: the device to use for pytorch computations.
If not provided, the devices of x_mat and y_mat are used.
:return: Tensor of shape (n_vectors, n_vectors) containing all pairwise
euclidian distances.
The index [i_x, i_y] contains the euclidian distance between
x_mat[i_x] and y_mat[i_y].
"""
...
Back
|
FazBrowse Home
|
New Git URL