/***************************************************************************
* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay *
* Copyright (c) QuantStack *
* *
* Distributed under the terms of the BSD 3-Clause License. *
* *
* The full license is in the file LICENSE, distributed with this software. *
****************************************************************************/
#ifndef PY_ARRAY_HPP
#define PY_ARRAY_HPP
#include
#include
#include
#include "xtensor/containers/xbuffer_adaptor.hpp"
#include "xtensor/core/xiterator.hpp"
#include "xtensor/core/xsemantic.hpp"
#include "pyarray_backstrides.hpp"
#include "pycontainer.hpp"
#include "pystrides_adaptor.hpp"
#include "pynative_casters.hpp"
#include "xtensor_type_caster_base.hpp"
#include "xtensor_python_config.hpp"
namespace xt
{
template
class pyarray;
}
namespace pybind11
{
namespace detail
{
#ifdef PYBIND11_DESCR // The macro is removed from pybind11 since 2.3
template
struct handle_type_name
{
static PYBIND11_DESCR name()
{
return _("numpy.ndarray[") + npy_format_descriptor::name() + _("]");
}
};
#endif
template
struct pyobject_caster
{
using type = xt::pyarray;
bool load(handle src, bool convert)
{
if (!convert)
{
if (!xt::detail::check_array(src))
{
return false;
}
}
value = type::ensure(src);
return static_cast(value);
}
static handle cast(const handle& src, return_value_policy, handle)
{
return src.inc_ref();
}
#ifdef PYBIND11_DESCR // The macro is removed from pybind11 since 2.3
PYBIND11_TYPE_CASTER(type, handle_type_name::name());
#else
PYBIND11_TYPE_CASTER(type, _("numpy.ndarray[") + npy_format_descriptor::name + _("]"));
#endif
};
// Type caster for casting ndarray to xexpression
template
struct type_caster : pyobject_caster
{
using Type = xt::xexpression;
operator Type&()
{
return this->value;
}
operator const Type&()
{
return this->value;
}
};
}
}
namespace xt
{
template
struct xiterable_inner_types
: xcontainer_iterable_types
{
};
template
struct xcontainer_inner_types
{
using storage_type = xbuffer_adaptor;
using reference = typename storage_type::reference;
using const_reference = typename storage_type::const_reference;
using size_type = typename storage_type::size_type;
using shape_type = std::vector;
using strides_type = std::vector;
using backstrides_type = pyarray_backstrides;
using inner_shape_type = xbuffer_adaptor;
using inner_strides_type = pystrides_adaptor;
using inner_backstrides_type = backstrides_type;
using temporary_type = pyarray;
static constexpr layout_type layout = L;
};
/**
* @class pyarray
* @brief Multidimensional container providing the xtensor container semantics to a numpy array.
*
* pyarray is similar to the xarray container in that it has a dynamic dimensionality.
* Reshapes of a pyarray container are reflected in the underlying numpy array.
*
* @tparam T The type of the element stored in the pyarray.
* @tparam L Static layout of the pyarray
*
* @sa pytensor
*/
template
class pyarray : public pycontainer,
public xcontainer_semantic
{
public:
using self_type = pyarray;
using semantic_base = xcontainer_semantic;
using base_type = pycontainer;
using storage_type = typename base_type::storage_type;
using value_type = typename base_type::value_type;
using reference = typename base_type::reference;
using const_reference = typename base_type::const_reference;
using pointer = typename base_type::pointer;
using size_type = typename base_type::size_type;
using difference_type = typename base_type::difference_type;
using shape_type = typename base_type::shape_type;
using strides_type = typename base_type::strides_type;
using backstrides_type = typename base_type::backstrides_type;
using inner_shape_type = typename base_type::inner_shape_type;
using inner_strides_type = typename base_type::inner_strides_type;
using inner_backstrides_type = typename base_type::inner_backstrides_type;
constexpr static std::size_t rank = SIZE_MAX;
pyarray();
pyarray(const value_type& t);
pyarray(nested_initializer_list_t t);
pyarray(nested_initializer_list_t t);
pyarray(nested_initializer_list_t t);
pyarray(nested_initializer_list_t t);
pyarray(nested_initializer_list_t t);
pyarray(pybind11::handle h, pybind11::object::borrowed_t);
pyarray(pybind11::handle h, pybind11::object::stolen_t);
pyarray(const pybind11::object& o);
explicit pyarray(const shape_type& shape, layout_type l = layout_type::row_major);
explicit pyarray(const shape_type& shape, const_reference value, layout_type l = layout_type::row_major);
explicit pyarray(const shape_type& shape, const strides_type& strides, const_reference value);
explicit pyarray(const shape_type& shape, const strides_type& strides);
template
static pyarray from_shape(S&& s);
pyarray(const self_type& rhs);
self_type& operator=(const self_type& rhs);
pyarray(self_type&&) = default;
self_type& operator=(self_type&& e) = default;
template
pyarray(const xexpression& e);
template
self_type& operator=(const xexpression& e);
using base_type::begin;
using base_type::end;
static self_type ensure(pybind11::handle h);
static bool check_(pybind11::handle h);
#if (PYBIND11_VERSION_MAJOR == 2 && PYBIND11_VERSION_MINOR >= 3) || PYBIND11_VERSION_MAJOR >= 3
// Prevent ambiguous overload resolution for operators defined for
// both xt::xcontainer_semantic and pybind11::object.
using semantic_base::operator+=;
using semantic_base::operator-=;
using semantic_base::operator*=;
using semantic_base::operator/=;
using semantic_base::operator|=;
using semantic_base::operator&=;
using semantic_base::operator^=;
// using semantic_base::operator=;
#endif
private:
inner_shape_type m_shape;
inner_strides_type m_strides;
mutable inner_backstrides_type m_backstrides;
storage_type m_storage;
void init_array(const shape_type& shape, const strides_type& strides);
void init_from_python();
const inner_shape_type& shape_impl() const noexcept;
const inner_strides_type& strides_impl() const noexcept;
const inner_backstrides_type& backstrides_impl() const noexcept;
storage_type& storage_impl() noexcept;
const storage_type& storage_impl() const noexcept;
layout_type default_dynamic_layout();
friend class xcontainer;
friend class pycontainer;
};
/**************************
* pyarray implementation *
**************************/
/**
* @name Constructors
*/
//@{
template
inline pyarray::pyarray()
: base_type()
{
// TODO: avoid allocation
shape_type shape = xtl::make_sequence(0, size_type(1));
strides_type strides = xtl::make_sequence(0, size_type(0));
init_array(shape, strides);
detail::default_initialize(m_storage);
}
/**
* Allocates a pyarray with nested initializer lists.
*/
template
inline pyarray::pyarray(const value_type& t)
: base_type()
{
base_type::resize(xt::shape(t), default_dynamic_layout());
nested_copy(m_storage.begin(), t);
}
template
inline pyarray::pyarray(nested_initializer_list_t t)
: base_type()
{
base_type::resize(xt::shape(t), default_dynamic_layout());
L == layout_type::row_major ? nested_copy(m_storage.begin(), t) : nested_copy(this->template begin(), t);
}
template
inline pyarray::pyarray(nested_initializer_list_t t)
: base_type()
{
base_type::resize(xt::shape(t), default_dynamic_layout());
L == layout_type::row_major ? nested_copy(m_storage.begin(), t) : nested_copy(this->template begin(), t);
}
template
inline pyarray::pyarray(nested_initializer_list_t t)
: base_type()
{
base_type::resize(xt::shape(t), default_dynamic_layout());
L == layout_type::row_major ? nested_copy(m_storage.begin(), t) : nested_copy(this->template begin(), t);
}
template
inline pyarray::pyarray(nested_initializer_list_t t)
: base_type()
{
base_type::resize(xt::shape(t), default_dynamic_layout());
L == layout_type::row_major ? nested_copy(m_storage.begin(), t) : nested_copy(this->template begin(), t);
}
template
inline pyarray::pyarray(nested_initializer_list_t t)
: base_type()
{
base_type::resize(xt::shape(t), default_dynamic_layout());
L == layout_type::row_major ? nested_copy(m_storage.begin(), t) : nested_copy(this->template begin(), t);
}
template
inline pyarray::pyarray(pybind11::handle h, pybind11::object::borrowed_t b)
: base_type(h, b)
{
init_from_python();
}
template
inline pyarray::pyarray(pybind11::handle h, pybind11::object::stolen_t s)
: base_type(h, s)
{
init_from_python();
}
template
inline pyarray::pyarray(const pybind11::object& o)
: base_type(o)
{
init_from_python();
}
/**
* Allocates an uninitialized pyarray with the specified shape and
* layout.
* @param shape the shape of the pyarray
* @param l the layout of the pyarray
*/
template
inline pyarray::pyarray(const shape_type& shape, layout_type l)
: base_type()
{
strides_type strides(shape.size());
compute_strides(shape, l, strides);
init_array(shape, strides);
}
/**
* Allocates a pyarray with the specified shape and layout. Elements
* are initialized to the specified value.
* @param shape the shape of the pyarray
* @param value the value of the elements
* @param l the layout of the pyarray
*/
template
inline pyarray::pyarray(const shape_type& shape, const_reference value, layout_type l)
: base_type()
{
strides_type strides(shape.size());
compute_strides(shape, l, strides);
init_array(shape, strides);
std::fill(m_storage.begin(), m_storage.end(), value);
}
/**
* Allocates an uninitialized pyarray with the specified shape and strides.
* Elements are initialized to the specified value.
* @param shape the shape of the pyarray
* @param strides the strides of the pyarray
* @param value the value of the elements
*/
template
inline pyarray::pyarray(const shape_type& shape, const strides_type& strides, const_reference value)
: base_type()
{
init_array(shape, strides);
std::fill(m_storage.begin(), m_storage.end(), value);
}
/**
* Allocates an uninitialized pyarray with the specified shape and strides.
* @param shape the shape of the pyarray
* @param strides the strides of the pyarray
*/
template
inline pyarray::pyarray(const shape_type& shape, const strides_type& strides)
: base_type()
{
init_array(shape, strides);
}
/**
* Allocates and returns an pyarray with the specified shape.
* @param shape the shape of the pyarray
*/
template
template
inline pyarray pyarray::from_shape(S&& shape)
{
auto shp = xtl::forward_sequence(shape);
return self_type(shp);
}
//@}
/**
* @name Copy semantic
*/
//@{
/**
* The copy constructor.
*/
template
inline pyarray::pyarray(const self_type& rhs)
: base_type(), semantic_base(rhs)
{
auto tmp = pybind11::reinterpret_steal(
PyArray_NewLikeArray(rhs.python_array(), NPY_KEEPORDER, nullptr, 1));
if (!tmp)
{
throw std::runtime_error("NumPy: unable to create ndarray");
}
this->m_ptr = tmp.release().ptr();
init_from_python();
std::copy(rhs.storage().cbegin(), rhs.storage().cend(), this->storage().begin());
}
/**
* The assignment operator.
*/
template
inline auto pyarray::operator=(const self_type& rhs) -> self_type&
{
self_type tmp(rhs);
*this = std::move(tmp);
return *this;
}
//@}
/**
* @name Extended copy semantic
*/
//@{
/**
* The extended copy constructor.
*/
template
template
inline pyarray::pyarray(const xexpression& e)
: base_type()
{
// TODO: prevent intermediary shape allocation
shape_type shape = xtl::forward_sequence(e.derived_cast().shape());
strides_type strides = xtl::make_sequence(shape.size(), size_type(0));
layout_type layout = default_dynamic_layout();
compute_strides(shape, layout, strides);
init_array(shape, strides);
semantic_base::assign(e);
}
/**
* The extended assignment operator.
*/
template
template
inline auto pyarray::operator=(const xexpression& e) -> self_type&
{
return semantic_base::operator=(e);
}
//@}
template
inline auto pyarray::ensure(pybind11::handle h) -> self_type
{
return base_type::ensure(h);
}
template
inline bool pyarray::check_(pybind11::handle h)
{
return base_type::check_(h);
}
template
inline void pyarray::init_array(const shape_type& shape, const strides_type& strides)
{
strides_type adapted_strides(strides);
std::transform(strides.begin(), strides.end(), adapted_strides.begin(),
[](auto v) { return sizeof(T) * v; });
int flags = NPY_ARRAY_ALIGNED;
if (!std::is_const::value)
{
flags |= NPY_ARRAY_WRITEABLE;
}
auto dtype = pybind11::detail::npy_format_descriptor::dtype();
npy_intp* shape_data = reinterpret_cast(const_cast(shape.data()));
npy_intp* strides_data = reinterpret_cast(adapted_strides.data());
auto tmp = pybind11::reinterpret_steal(
PyArray_NewFromDescr(&PyArray_Type, (PyArray_Descr*) dtype.release().ptr(), static_cast(shape.size()), shape_data, strides_data,
nullptr, flags, nullptr));
if (!tmp)
{
throw std::runtime_error("NumPy: unable to create ndarray");
}
this->m_ptr = tmp.release().ptr();
init_from_python();
}
template
inline void pyarray::init_from_python()
{
if (!static_cast(*this))
{
return;
}
m_shape = inner_shape_type(reinterpret_cast(PyArray_SHAPE(this->python_array())),
static_cast(PyArray_NDIM(this->python_array())));
m_strides = inner_strides_type(reinterpret_cast(PyArray_STRIDES(this->python_array())),
static_cast(PyArray_NDIM(this->python_array())),
reinterpret_cast(PyArray_SHAPE(this->python_array())));
if (L != layout_type::dynamic && !do_strides_match(m_shape, m_strides, L, 1))
{
throw std::runtime_error("NumPy: passing container with bad strides for layout (is it a view?).");
}
m_backstrides = backstrides_type(*this);
m_storage = storage_type(reinterpret_cast(PyArray_DATA(this->python_array())),
this->get_buffer_size());
}
template
inline auto pyarray::shape_impl() const noexcept -> const inner_shape_type&
{
return m_shape;
}
template
inline auto pyarray::strides_impl() const noexcept -> const inner_strides_type&
{
return m_strides;
}
template
inline auto pyarray::backstrides_impl() const noexcept -> const inner_backstrides_type&
{
// m_backstrides wraps the numpy array backstrides, which is a raw pointer.
// The address of the raw pointer stored in the wrapper would be invalidated when the pyarray is copied.
// Hence, we build a new backstrides object (cheap wrapper around the underlying pointer) upon access.
m_backstrides = backstrides_type(*this);
return m_backstrides;
}
template
inline auto pyarray::storage_impl() noexcept -> storage_type&
{
return m_storage;
}
template
inline auto pyarray::storage_impl() const noexcept -> const storage_type&
{
return m_storage;
}
template
layout_type pyarray::default_dynamic_layout()
{
return L == layout_type::dynamic ? layout_type::row_major : L;
}
}
#endif