# Python wrapper for emscripten_* C functions - Fetch API
# Copyright (C) 2019 Sylvain Beucler
# Copying and distribution of this file, with or without modification,
# are permitted in any medium without royalty provided the copyright
# notice and this notice are preserved. This file is offered as-is,
# without any warranty.
# http://docs.cython.org/en/latest/src/tutorial/strings.html#auto-encoding-and-decoding
# Most of our strings are converted from/to JS through emscripten stringToUTF8/UTF8ToString
# cython: c_string_type=unicode, c_string_encoding=utf8
# Note: Py->C auto-UTF-8 (instead of .encode('UTF-8')) not supported for Py2
# Note: causes issues with Typed Memoryviews
# TODO: move to emscripten.fetch but requires emscripten/__init__.py,
# patching emscripten_fetch.c, etc.
from __future__ import print_function
from cpython.mem cimport PyMem_Malloc, PyMem_Free
from cpython.ref cimport PyObject, Py_XINCREF, Py_XDECREF
from cpython.buffer cimport PyBuffer_FillInfo
from libc.string cimport strncpy
from libc.stdint cimport uint32_t, uint64_t
cdef extern from "emscripten/html5.h":
ctypedef int EM_BOOL
ctypedef int EMSCRIPTEN_RESULT
enum: EM_TRUE
enum: EM_FALSE
cdef extern from "emscripten/fetch.h":
ctypedef struct emscripten_fetch_attr_t:
char requestMethod[32]
void *userData
void (*onsuccess)(emscripten_fetch_t *fetch)
void (*onerror)(emscripten_fetch_t *fetch)
void (*onprogress)(emscripten_fetch_t *fetch)
void (*onreadystatechange)(emscripten_fetch_t *fetch)
uint32_t attributes
unsigned long timeoutMSecs
EM_BOOL withCredentials
const char *destinationPath
const char *userName
const char *password
const char * const *requestHeaders
const char *overriddenMimeType
const char *requestData
size_t requestDataSize
ctypedef struct emscripten_fetch_t:
unsigned int id
void *userData
const char *url
const char *data
uint64_t numBytes
uint64_t dataOffset
uint64_t totalBytes
unsigned short readyState
unsigned short status
char statusText[64]
uint32_t __proxyState
emscripten_fetch_attr_t __attributes
enum:
EMSCRIPTEN_FETCH_LOAD_TO_MEMORY
EMSCRIPTEN_FETCH_STREAM_DATA
EMSCRIPTEN_FETCH_PERSIST_FILE
EMSCRIPTEN_FETCH_APPEND
EMSCRIPTEN_FETCH_REPLACE
EMSCRIPTEN_FETCH_NO_DOWNLOAD
EMSCRIPTEN_FETCH_SYNCHRONOUS
EMSCRIPTEN_FETCH_WAITABLE
void emscripten_fetch_attr_init(emscripten_fetch_attr_t *fetch_attr)
emscripten_fetch_t *emscripten_fetch(emscripten_fetch_attr_t *fetch_attr, const char *url)
#EMSCRIPTEN_RESULT emscripten_fetch_wait(emscripten_fetch_t *fetch, double timeoutMSecs)
EMSCRIPTEN_RESULT emscripten_fetch_close(emscripten_fetch_t *fetch)
size_t emscripten_fetch_get_response_headers_length(emscripten_fetch_t *fetch)
size_t emscripten_fetch_get_response_headers(emscripten_fetch_t *fetch, char *dst, size_t dstSizeBytes)
char **emscripten_fetch_unpack_response_headers(const char *headersString)
void emscripten_fetch_free_unpacked_response_headers(char **unpackedHeaders)
LOAD_TO_MEMORY = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY
STREAM_DATA = EMSCRIPTEN_FETCH_STREAM_DATA
PERSIST_FILE = EMSCRIPTEN_FETCH_PERSIST_FILE
APPEND = EMSCRIPTEN_FETCH_APPEND
REPLACE = EMSCRIPTEN_FETCH_REPLACE
NO_DOWNLOAD = EMSCRIPTEN_FETCH_NO_DOWNLOAD
SYNCHRONOUS = EMSCRIPTEN_FETCH_SYNCHRONOUS
WAITABLE = EMSCRIPTEN_FETCH_WAITABLE
# Fetch API
# https://emscripten.org/docs/api_reference/fetch.html
# http://docs.cython.org/en/latest/src/userguide/extension_types.html
cdef class Fetch:
cdef emscripten_fetch_t *fetch
cdef callbacks
def __cinit__(self, url, requestMethod=None, userData=None,
onsuccess=None, onerror=None, onprogress=None, onreadystatechange=None,
attributes=None, timeoutMSecs=None, withCredentials=None,
destinationPath=None, userName=None, password=None,
requestHeaders=None, overriddenMimeType=None, requestData=None):
# Keep track of temporary Python strings we pass emscripten_fetch() for copy
py_str_refs = []
cdef emscripten_fetch_attr_t attr
emscripten_fetch_attr_init(&attr)
Py_XINCREF(self) # survive until callback
attr.userData = self
if requestMethod is not None:
strncpy(attr.requestMethod,
requestMethod.encode('UTF-8'),
sizeof(attr.requestMethod) - 1)
self.userData = userData
self.callbacks = {}
attr.onsuccess = callpyfunc_fetch_onsuccess
attr.onerror = callpyfunc_fetch_onerror
if onsuccess is not None:
self.callbacks['onsuccess'] = onsuccess
if onerror is not None:
self.callbacks['onerror'] = onerror
if onprogress is not None:
self.callbacks['onprogress'] = onprogress
attr.onprogress = callpyfunc_fetch_onprogress
if onreadystatechange is not None:
self.callbacks['onreadystatechange'] = onreadystatechange
attr.onreadystatechange = callpyfunc_fetch_onreadystatechange
if attributes is not None:
attr.attributes = attributes
if timeoutMSecs is not None:
attr.timeoutMSecs = timeoutMSecs
if withCredentials is not None:
attr.withCredentials = withCredentials
if destinationPath is not None:
py_str_refs.append(destinationPath.encode('UTF-8'))
attr.destinationPath = py_str_refs[-1]
if userName is not None:
py_str_refs.append(userName.encode('UTF-8'))
attr.userName = py_str_refs[-1]
if password is not None:
py_str_refs.append(password.encode('UTF-8'))
attr.password = py_str_refs[-1]
cdef char** headers
if requestHeaders is not None:
size = (2 * len(requestHeaders) + 1) * sizeof(char*)
headers = PyMem_Malloc(size)
i = 0
for name,value in requestHeaders.items():
py_str_refs.append(name.encode('UTF-8'))
headers[i] = py_str_refs[-1]
i += 1
py_str_refs.append(value.encode('UTF-8'))
headers[i] = py_str_refs[-1]
i += 1
headers[i] = NULL
attr.requestHeaders = headers
if overriddenMimeType is not None:
py_str_refs.append(overriddenMimeType.encode('UTF-8'))
attr.overriddenMimeType = py_str_refs[-1]
if requestData is not None:
size = len(requestData)
attr.requestDataSize = size
# direct pointer, no UTF-8 encoding pass:
attr.requestData = requestData
# Fetch
cdef emscripten_fetch_t *fetch = emscripten_fetch(&attr, url.encode('UTF-8'))
self.fetch = fetch
# Explicitely deref temporary Python strings. Test for forgotten refs with e.g.:
# print(attr.overriddenMimeType, attr.destinationPath, attr.userName, attr.password)
del py_str_refs
if requestHeaders is not None:
PyMem_Free(attr.requestHeaders)
def __dealloc__(self):
emscripten_fetch_close(self.fetch)
# Currently unsafe:
# https://github.com/emscripten-core/emscripten/issues/8234
#def fetch_close(fetch):
# pass
# http://docs.cython.org/en/latest/src/userguide/buffer.html
# https://docs.python.org/3/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer
# https://docs.python.org/3/c-api/buffer.html#c.PyObject_GetBuffer
# https://docs.python.org/3/c-api/buffer.html#c.PyBuffer_FillInfo
def __getbuffer__(self, Py_buffer *view, int flags):
if self.fetch.data != NULL:
is_readonly = 1
PyBuffer_FillInfo(view, self, self.fetch.data, self.fetch.numBytes, is_readonly, flags)
else:
view.obj = None
raise BufferError
def __releasebuffer__(self, Py_buffer *view):
pass
def __repr__(self):
return u''.format(repr(self.id), repr(self.userData), repr(self.url), self.data and "" or "None", repr(self.numBytes), repr(self.dataOffset), repr(self.totalBytes), repr(self.readyState), repr(self.status), repr(self.statusText))
# For testing whether a copy occurred:
#def overwrite(self):
# cdef char* overwrite = (self.fetch.data)
# overwrite[0] = b'O'
def get_response_headers(self):
cdef char* buf = NULL
# Note: JS crash if applied on a persisted request from IDB cache
# https://github.com/emscripten-core/emscripten/issues/7026#issuecomment-545488132
cdef length = emscripten_fetch_get_response_headers_length(self.fetch)
if length > 0:
headersString = PyMem_Malloc(length)
emscripten_fetch_get_response_headers(self.fetch, headersString, length+1)
ret = headersString[:length] # copy
PyMem_Free(headersString)
return ret
else:
return None
def get_unpacked_response_headers(self):
cdef char* headersString = NULL
cdef char** unpackedHeaders = NULL
# Note: JS crash if applied on a persisted request from IDB cache
cdef length = emscripten_fetch_get_response_headers_length(self.fetch)
if length > 0:
headersString = PyMem_Malloc(length)
emscripten_fetch_get_response_headers(self.fetch, headersString, length+1)
unpackedHeaders = emscripten_fetch_unpack_response_headers(headersString)
PyMem_Free(headersString)
d = {}
i = 0
while unpackedHeaders[i] != NULL:
k = unpackedHeaders[i] # c_string_encoding
i += 1
v = unpackedHeaders[i] # c_string_encoding
i += 1
d[k] = v
emscripten_fetch_free_unpacked_response_headers(unpackedHeaders)
return d
else:
return None
@property
def id(self):
return self.fetch.id
cdef readonly userData
@property
def url(self):
#return self.fetch.url.decode('UTF-8')
return self.fetch.url # c_string_encoding
@property
def data(self):
if self.fetch.data != NULL:
return self
else:
return None
@property
def numBytes(self):
return self.fetch.numBytes
@property
def dataOffset(self):
return self.fetch.dataOffset
@property
def totalBytes(self):
return self.fetch.totalBytes # Content-Length
@property
def readyState(self):
return self.fetch.readyState
@property
def status(self):
return self.fetch.status
@property
def statusText(self):
#return self.fetch.statusText.decode('UTF-8')
return self.fetch.statusText # c_string_encoding
cdef void callpyfunc_fetch_callback(emscripten_fetch_t *fetch, char* callback_name):
cdef Fetch py_fetch = fetch.userData
# for theoretical concurrency, if we're called during emscripten_fetch()
py_fetch.fetch = fetch
# call Python function
if py_fetch.callbacks.get(callback_name, None):
py_fetch.callbacks[callback_name](py_fetch)
# one of {onsuccess,onerror} is guaranteed to run, deref Fetch there
cdef void callpyfunc_fetch_onsuccess(emscripten_fetch_t *fetch):
callpyfunc_fetch_callback(fetch, 'onsuccess')
Py_XDECREF(fetch.userData)
cdef void callpyfunc_fetch_onerror(emscripten_fetch_t *fetch):
callpyfunc_fetch_callback(fetch, 'onerror')
Py_XDECREF(fetch.userData)
cdef void callpyfunc_fetch_onprogress(emscripten_fetch_t *fetch):
callpyfunc_fetch_callback(fetch, 'onprogress')
cdef void callpyfunc_fetch_onreadystatechange(emscripten_fetch_t *fetch):
callpyfunc_fetch_callback(fetch, 'onreadystatechange')
# import emscripten_fetch,sys; f=lambda x:sys.stdout.write(repr(x)+"\n");
# #Module.cwrap('PyRun_SimpleString', 'number', ['string'])("def g(x):\n global a; a=x")
# emscripten_fetch.Fetch('/', onsuccess=f)
# emscripten_fetch.Fetch(u'/hello', onsuccess=f)
# emscripten_fetch.Fetch('/hello', attributes=emscripten_fetch.LOAD_TO_MEMORY, onsuccess=f); del f # output
# fetch_attr={'onsuccess':f}; emscripten_fetch.Fetch('/hello', **fetch_attr); del fetch_attr['onsuccess'] # output
# emscripten_fetch.Fetch('/non-existent', onerror=lambda x:sys.stdout.write(repr(x)+"\n"))
# emscripten_fetch.Fetch('https://bank.confidential/', onerror=lambda x:sys.stdout.write(repr(x)+"\n")) # simulated 404
# emscripten_fetch.Fetch('/hello', attributes=emscripten_fetch.LOAD_TO_MEMORY|emscripten_fetch.PERSIST_FILE, onsuccess=f)
# Note: fe.fetch.id changes (in-place) when first caching
# emscripten_fetch.Fetch('/hello', requestMethod='EM_IDB_DELETE', onsuccess=f)
# emscripten_fetch.Fetch('/hello', attributes=emscripten_fetch.LOAD_TO_MEMORY, requestMethod='POST', requestData='AA\xffBB\x00CC', onsuccess=f, onerror=f)
# emscripten_fetch.Fetch('/hello', attributes=emscripten_fetch.LOAD_TO_MEMORY, requestMethod='12345678901234567890123456789012', onerror=f)
# emscripten_fetch.Fetch('/hello', attributes=emscripten_fetch.LOAD_TO_MEMORY, onsuccess=f, userData='userData', overriddenMimeType='text/html', userName='userName', password='password', requestHeaders={'Content-Type':'text/plain','Cache-Control':'no-store'})
# emscripten_fetch.Fetch('/hello', attributes=emscripten_fetch.LOAD_TO_MEMORY|emscripten_fetch.PERSIST_FILE, onsuccess=f, destinationPath='destinationPath'); emscripten_fetch.Fetch('destinationPath', requestMethod='EM_IDB_DELETE', onsuccess=f)
# fe=emscripten_fetch.Fetch('/hello', attributes=emscripten_fetch.LOAD_TO_MEMORY|emscripten_fetch.PERSIST_FILE, onsuccess=f, destinationPath='destinationPath'); fe2=emscripten_fetch.Fetch('destinationPath', requestMethod='EM_IDB_DELETE', onsuccess=f); print("fe=",fe); print("fe2=",fe2)
# Note: fe2 can occur before fe1
# r=emscripten_fetch.Fetch('/hello', attributes=emscripten_fetch.LOAD_TO_MEMORY)
# open('test.txt','wb').write(r); open('test.txt','rb').read()
# r.data != None
# memoryview(r)[:5].tobytes()
# import cStringIO; cStringIO.StringIO(r).read(5)