:param protocol: the protocol to use, e.g. 'file', 's3'
:return: the backend class
"""
ifprotocolincls.__backends__:
returncls.__backends__[protocol]
ifprotocol=='file':
fromdocarray.store.fileimportFileDocStore
cls.__backends__[protocol] =FileDocStore
logging.debug('Loaded Local Filesystem backend')
elifprotocol=='s3':
fromdocarray.store.s3importS3DocStore
cls.__backends__[protocol] =S3DocStore
logging.debug('Loaded S3 backend')
else:
raiseNotImplementedError(f'protocol {protocol} not supported')
returncls.__backends__[protocol]
defpush(
self,
url: str,
show_progress: bool=False,
**kwargs,
) ->Dict:
"""Push this `DocList` object to the specified url.
:param url: url specifying the protocol and save name of the `DocList`. Should be of the form ``protocol://namespace/name``. e.g. ``s3://bucket/path/to/namespace/name``, ``file:///path/to/folder/name``
:param show_progress: If true, a progress bar will be displayed.
"""
logging.info(f'Pushing {len(self)} docs to {url}')
"""Push a stream of documents to the specified url.
:param docs: a stream of documents
:param url: url specifying the protocol and save name of the `DocList`. Should be of the form ``protocol://namespace/name``. e.g. ``s3://bucket/path/to/namespace/name``, ``file:///path/to/folder/name``
:param show_progress: If true, a progress bar will be displayed.
:param url: url specifying the protocol and save name of the `DocList`. Should be of the form ``protocol://namespace/name``. e.g. ``s3://bucket/path/to/namespace/name``, ``file:///path/to/folder/name``
:param show_progress: if true, display a progress bar.
:param local_cache: store the downloaded `DocList` to local folder
:return: a `DocList` object
"""
fromdocarray.base_docimportAnyDoc
ifcls.doc_type==AnyDoc:
raiseTypeError(
'There is no document schema defined. '
'Please specify the `DocList`\'s Document type using `DocList[MyDoc]`.'
"""Pull a stream of Documents from the specified url.
:param url: url specifying the protocol and save name of the `DocList`. Should be of the form ``protocol://namespace/name``. e.g. ``s3://bucket/path/to/namespace/name``, ``file:///path/to/folder/name``
:param show_progress: if true, display a progress bar.
:param local_cache: store the downloaded `DocList` to local folder
:return: Iterator of Documents
"""
fromdocarray.base_docimportAnyDoc
ifcls.doc_type==AnyDoc:
raiseTypeError(
'There is no document schema defined. '
'Please specify the `DocList`\'s Document type using `DocList[MyDoc]`.'
)
logging.info(f'Pulling Document stream from {url}')