Merge pull request #240 from jcarpent/devel

Fix contruction of np.array from custom types

cmake @ d43ba7b6

-Subproject commit 094834088ad9de32f1abdc2d56d29bca2190772c
+Subproject commit d43ba7b63f2ead6899665156eecfba96c9078239

include/eigenpy/numpy.hpp

 /*
- * Copyright 2020 INRIA
+ * Copyright 2020-2021 INRIA
  */
 
 #ifndef __eigenpy_numpy_hpp__
@@ -43,7 +43,13 @@ namespace eigenpy
   template <> struct NumpyEquivalentType< std::complex<long double> >  { enum { type_code = NPY_CLONGDOUBLE };};
   template <> struct NumpyEquivalentType<bool>    { enum { type_code = NPY_BOOL  };};
   template <> struct NumpyEquivalentType<int>     { enum { type_code = NPY_INT    };};
-  template <> struct NumpyEquivalentType<long>    { enum { type_code = NPY_LONG    };};
+  template <> struct NumpyEquivalentType<unsigned int>     { enum { type_code = NPY_UINT    };};
+#if __APPLE__
+  template <> struct NumpyEquivalentType<long>    { enum { type_code = NPY_INT64    };};
+#endif
+  template <> struct NumpyEquivalentType<unsigned long>    { enum { type_code = NPY_ULONG    };};
+  template <> struct NumpyEquivalentType<int64_t>    { enum { type_code = NPY_INT64 };};
+//  template <> struct NumpyEquivalentType<long long>    { enum { type_code = NPY_LONGLONG };};
 
   template<typename Scalar>
   bool isNumpyNativeType()
@@ -77,6 +83,8 @@ namespace eigenpy
   EIGENPY_DLLAPI int call_PyArray_RegisterCanCast(PyArray_Descr *descr, int totype, NPY_SCALARKIND scalar);
 
   EIGENPY_DLLAPI PyArray_Descr * call_PyArray_MinScalarType(PyArrayObject *arr);
+
+  EIGENPY_DLLAPI int call_PyArray_RegisterCastFunc(PyArray_Descr* descr, int totype, PyArray_VectorUnaryFunc* castfunc);
 }
 #else
   #define call_PyArray_Check(py_obj) PyArray_Check(py_obj)
@@ -89,6 +97,7 @@ namespace eigenpy
   #define call_PyArray_InitArrFuncs(funcs) PyArray_InitArrFuncs(funcs)
   #define call_PyArray_RegisterDataType(dtype) PyArray_RegisterDataType(dtype)
   #define call_PyArray_RegisterCanCast(descr,totype,scalar) PyArray_RegisterCanCast(descr,totype,scalar)
+  #define call_PyArray_RegisterCastFunc(descr,totype,castfunc) PyArray_RegisterCastFunc(descr,totype,castfunc)
 #endif
 
 #endif // ifndef __eigenpy_numpy_hpp__

include/eigenpy/register.hpp

@@ -56,6 +56,20 @@ namespace eigenpy
       }
     }
     
+    template<typename Scalar>
+    static PyArray_Descr * getPyArrayDescr()
+    {
+      namespace bp = boost::python;
+      if(!isNumpyNativeType<Scalar>())
+      {
+        return getPyArrayDescr(getPyType<Scalar>());
+      }
+      else
+      {
+        return call_PyArray_DescrFromType(NumpyEquivalentType<Scalar>::type_code);
+      }
+    }
+    
     template<typename Scalar>
     static int getTypeCode()
     {
@@ -79,12 +93,15 @@ namespace eigenpy
     static int registerNewType(PyTypeObject * py_type_ptr,
                                const std::type_info * type_info_ptr,
                                const int type_size,
+                               const int alignment,
                                PyArray_GetItemFunc * getitem,
                                PyArray_SetItemFunc * setitem,
                                PyArray_NonzeroFunc * nonzero,
                                PyArray_CopySwapFunc * copyswap,
                                PyArray_CopySwapNFunc * copyswapn,
-                               PyArray_DotFunc * dotfunc);
+                               PyArray_DotFunc * dotfunc,
+                               PyArray_FillFunc * fill,
+                               PyArray_FillWithScalarFunc * fillwithscalar);
     
     static Register & instance();
     

include/eigenpy/user-type.hpp

 //
-// Copyright (c) 2020 INRIA
+// Copyright (c) 2020-2021 INRIA
 //
 
 #ifndef __eigenpy_user_type_hpp__
@@ -11,8 +11,32 @@
 
 namespace eigenpy
 {
+  /// \brief Default cast algo to cast a From to To. Can be specialized for any types.
+  template<typename From, typename To>
+  struct cast
+  {
+    static To run(const From & from)
+    {
+      return static_cast<To>(from);
+    }
+    
+  };
+
   namespace internal
   {
+  
+    template<typename From, typename To>
+    static void cast(void * from_, void * to_, npy_intp n, void * /*fromarr*/, void * /*toarr*/)
+    {
+//      std::cout << "cast::run" << std::endl;
+      const From* from = static_cast<From*>(from_);
+      To* to = static_cast<To*>(to_);
+      for(npy_intp i = 0; i < n; i++)
+      {
+        to[i] = eigenpy::cast<From,To>::run(from[i]);
+      }
+    }
+  
     template<typename T, int type_code = NumpyEquivalentType<T>::type_code>
     struct SpecialMethods
     {
@@ -24,7 +48,21 @@ namespace eigenpy
       inline static npy_bool nonzero(void * /*ip*/, void * /*array*/) /*{ return (npy_bool)false; }*/;
       inline static void dotfunc(void * /*ip0_*/, npy_intp /*is0*/, void * /*ip1_*/, npy_intp /*is1*/,
                           void * /*op*/, npy_intp /*n*/, void * /*arr*/);
-//      static void cast(void * /*from*/, void * /*to*/, npy_intp /*n*/, void * /*fromarr*/, void * /*toarr*/) {};
+      inline static int fill(void* data_, npy_intp length, void* arr);
+      inline static int fillwithscalar(void* buffer_, npy_intp length,
+                                       void* value, void* arr);
+    };
+  
+    template<typename T>
+    struct OffsetOf
+    {
+      struct Data
+      {
+        char c;
+        T v;
+      };
+      
+      enum { value = offsetof(Data, v) };
     };
   
     template<typename T>
@@ -47,26 +85,38 @@ namespace eigenpy
           std::swap(t1,t2);
         }
       }
-      
-      inline static PyObject * getitem(void * ip, void * ap)
+
+      ///
+      /// \brief Get a python object from an array
+      ///        It returns a standard Python object from
+      ///        a single element of the array object arr pointed to by data.
+      /// \param[in] data Pointer to the first element of the C++ data stream
+      /// \param[in] arr  Pointer to the first element of the Python object data stream
+      ///
+      /// \returns PyObject corresponding to the python datastream.
+      ///
+      inline static PyObject * getitem(void * ip, void * /*ap*/)
       {
 //        std::cout << "getitem" << std::endl;
-        PyArrayObject * py_array = static_cast<PyArrayObject *>(ap);
-        if((py_array==NULL) || PyArray_ISBEHAVED_RO(py_array))
-        {
-          T * elt_ptr = static_cast<T*>(ip);
-          bp::object m(boost::ref(*elt_ptr));
-          Py_INCREF(m.ptr());
-          return m.ptr();
-        }
-        else
-        {
-          T * elt_ptr = static_cast<T*>(ip);
-          bp::object m(boost::ref(*elt_ptr));
-          Py_INCREF(m.ptr());
-          return m.ptr();
-        }
+        T * elt_ptr = static_cast<T*>(ip);
+        bp::object m(*elt_ptr);
+        Py_INCREF(m.ptr());
+        return m.ptr();
       }
+
+      ///
+      /// \brief Set a python object in an array.
+      ///        It sets the Python object "item" into the array, arr, at the position
+      ///        pointed to by data. This function deals with “misbehaved” arrays.
+      ///        If successful, a zero is returned, otherwise, a negative one is returned
+      ///        (and a Python error set).
+      
+      /// \param[in] src_obj  Pointer to the location of the python object
+      /// \param[in] dest_ptr Pointer to the location in the array where the source object should be saved.
+      /// \param[in] array Pointer to the location of the array
+      ///
+      /// \returns int Success(0) or Failure(-1)
+      ///
       
       inline static int setitem(PyObject * src_obj, void * dest_ptr, void * array)
       {
@@ -83,6 +133,10 @@ namespace eigenpy
         
         if(array_scalar_type != src_obj_type)
         {
+          std::stringstream ss;
+          ss << "The input type is of wrong type. ";
+          ss << "The expected type is " << bp::type_info(typeid(T)).name() << std::endl;
+          eigenpy::Exception(ss.str());
           return -1;
         }
         
@@ -144,28 +198,100 @@ namespace eigenpy
       inline static void dotfunc(void * ip0_, npy_intp is0, void * ip1_, npy_intp is1,
                                  void * op, npy_intp n, void * /*arr*/)
       {
-          T res = T(0);
-          char *ip0 = (char*)ip0_, *ip1 = (char*)ip1_;
-          npy_intp i;
-          for(i = 0; i < n; i++)
-          {
-            
-            res += *static_cast<T*>(static_cast<void*>(ip0))
-            * *static_cast<T*>(static_cast<void*>(ip1));
-            ip0 += is0;
-            ip1 += is1;
-          }
-          *static_cast<T*>(op) = res;
+//        std::cout << "dotfunc" << std::endl;
+        T res(0);
+        char *ip0 = (char*)ip0_, *ip1 = (char*)ip1_;
+        npy_intp i;
+        for(i = 0; i < n; i++)
+        {
+          
+          res += *static_cast<T*>(static_cast<void*>(ip0))
+          * *static_cast<T*>(static_cast<void*>(ip1));
+          ip0 += is0;
+          ip1 += is1;
+        }
+        *static_cast<T*>(op) = res;
+      }
+      
+      
+      inline static int fillwithscalar(void* buffer_, npy_intp length,
+                                       void* value, void* /*arr*/)
+      {
+//        std::cout << "fillwithscalar" << std::endl;
+        T r = *static_cast<T*>(value);
+        T* buffer = static_cast<T*>(buffer_);
+        npy_intp i;
+        for (i = 0; i < length; i++) {
+          buffer[i] = r;
+        }
+        return 0;
+      }
+      
+      
+      static int fill(void* data_, npy_intp length, void* /*arr*/)
+      {
+//        std::cout << "fillwithscalar" << std::endl;
+        T* data = static_cast<T*>(data_);
+        const T delta = data[1] - data[0];
+        T r = data[1];
+        npy_intp i;
+        for (i = 2; i < length; i++) {
+          r = r + delta;
+          data[i] = r;
+        }
+        return 0;
       }
       
-//      static void cast(void * from, void * to, npy_intp n, void * fromarr, void * toarr)
-//      {
-//      }
 
-    };
+    };  //     struct SpecialMethods<T,NPY_USERDEF>
   
   } // namespace internal
 
+
+  template<typename From, typename To>
+  bool registerCast(const bool safe)
+  {
+    PyArray_Descr* from_array_descr = Register::getPyArrayDescr<From>();
+//    int from_typenum = Register::getTypeCode<From>();
+    
+//    PyTypeObject * to_py_type = Register::getPyType<To>();
+    int to_typenum = Register::getTypeCode<To>();
+    assert(to_typenum >= 0 && "to_typenum is not valid");
+    assert(from_array_descr != NULL && "from_array_descr is not valid");
+    
+    if(call_PyArray_RegisterCastFunc(from_array_descr,
+                                     to_typenum,
+                                     static_cast<PyArray_VectorUnaryFunc *>(&eigenpy::internal::cast<From,To>)) < 0)
+    {
+      std::stringstream ss;
+      ss
+      << "PyArray_RegisterCastFunc of the cast from "
+      << bp::type_info(typeid(From)).name()
+      << " to "
+      << bp::type_info(typeid(To)).name()
+      << " has failed.";
+      eigenpy::Exception(ss.str());
+      return false;
+    }
+    
+    if (safe && call_PyArray_RegisterCanCast(from_array_descr,
+                                             to_typenum,
+                                             NPY_NOSCALAR) < 0)
+    {
+      std::stringstream ss;
+      ss
+      << "PyArray_RegisterCanCast of the cast from "
+      << bp::type_info(typeid(From)).name()
+      << " to "
+      << bp::type_info(typeid(To)).name()
+      << " has failed.";
+      eigenpy::Exception(ss.str());
+      return false;
+    }
+    
+    return true;
+  }
+
   template<typename Scalar>
   int registerNewType(PyTypeObject * py_type_ptr = NULL)
   {
@@ -189,14 +315,18 @@ namespace eigenpy
     PyArray_CopySwapFunc * copyswap = &internal::SpecialMethods<Scalar>::copyswap;
     PyArray_CopySwapNFunc * copyswapn = reinterpret_cast<PyArray_CopySwapNFunc*>(&internal::SpecialMethods<Scalar>::copyswapn);
     PyArray_DotFunc * dotfunc = &internal::SpecialMethods<Scalar>::dotfunc;
-//    PyArray_CastFunc * cast = &internal::SpecialMethods<Scalar>::cast;
+    PyArray_FillFunc * fill = &internal::SpecialMethods<Scalar>::fill;
+    PyArray_FillWithScalarFunc * fillwithscalar = &internal::SpecialMethods<Scalar>::fillwithscalar;
     
     int code = Register::registerNewType(py_type_ptr,
                                          &typeid(Scalar),
                                          sizeof(Scalar),
+                                         internal::OffsetOf<Scalar>::value,
                                          getitem, setitem, nonzero,
                                          copyswap, copyswapn,
-                                         dotfunc);
+                                         dotfunc,
+                                         fill,
+                                         fillwithscalar);
     
     call_PyArray_RegisterCanCast(call_PyArray_DescrFromType(NPY_OBJECT),
                                  code, NPY_NOSCALAR);

src/numpy.cpp

 /*
- * Copyright 2020 INRIA
+ * Copyright 2020-2021 INRIA
  */
 
 #include "eigenpy/numpy.hpp"
@@ -68,6 +68,11 @@ namespace eigenpy
   {
     return PyArray_RegisterCanCast(descr,totype,scalar);
   }
-  
+
+  int call_PyArray_RegisterCastFunc(PyArray_Descr* descr, int totype, PyArray_VectorUnaryFunc* castfunc)
+  {
+    return PyArray_RegisterCastFunc(descr,totype,castfunc);
+  }
+
 #endif
 }

src/register.cpp

 /*
- * Copyright 2020 INRIA
+ * Copyright 2020-2021 INRIA
  */
 
 #include "eigenpy/register.hpp"
@@ -36,22 +36,43 @@ namespace eigenpy
   int Register::registerNewType(PyTypeObject * py_type_ptr,
                                 const std::type_info * type_info_ptr,
                                 const int type_size,
+                                const int alignement,
                                 PyArray_GetItemFunc * getitem,
                                 PyArray_SetItemFunc * setitem,
                                 PyArray_NonzeroFunc * nonzero,
                                 PyArray_CopySwapFunc * copyswap,
                                 PyArray_CopySwapNFunc * copyswapn,
-                                PyArray_DotFunc * dotfunc)
+                                PyArray_DotFunc * dotfunc,
+                                PyArray_FillFunc * fill,
+                                PyArray_FillWithScalarFunc * fillwithscalar)
   {
+    namespace bp = boost::python;
+
+    bp::tuple tp_bases_extended(bp::make_tuple(bp::handle<>(bp::borrowed(&PyGenericArrType_Type))));
+    tp_bases_extended += bp::tuple(bp::handle<>(bp::borrowed(py_type_ptr->tp_bases)));
+                                
+    Py_INCREF(tp_bases_extended.ptr());
+    py_type_ptr->tp_bases = tp_bases_extended.ptr();
+
+    py_type_ptr->tp_flags &= ~Py_TPFLAGS_READY; // to force the rebuild
+//    py_type_ptr->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE;
+    if(PyType_Ready(py_type_ptr) < 0) // Force rebuilding of the __bases__ and mro
+    {
+      throw std::invalid_argument("PyType_Ready fails to initialize input type.");
+    }
+
     PyArray_Descr * descr_ptr = new PyArray_Descr(*call_PyArray_DescrFromType(NPY_OBJECT));
     PyArray_Descr & descr = *descr_ptr;
     descr.typeobj = py_type_ptr;
     descr.kind = 'V';
     descr.byteorder = '=';
+    descr.type = 'r';
     descr.elsize = type_size;
-    descr.flags = NPY_LIST_PICKLE | NPY_USE_GETITEM | NPY_USE_SETITEM | NPY_NEEDS_INIT | NPY_NEEDS_PYAPI;
-    //      descr->names = PyTuple_New(0);
-    //      descr->fields = PyDict_New();
+    descr.flags = NPY_NEEDS_PYAPI | NPY_USE_GETITEM | NPY_USE_SETITEM | NPY_NEEDS_INIT;
+    descr.type_num = 0;
+    descr.names = 0;
+    descr.fields = 0;
+    descr.alignment = alignement; //call_PyArray_DescrFromType(NPY_OBJECT)->alignment;
     
     PyArray_ArrFuncs * funcs_ptr = new PyArray_ArrFuncs;
     PyArray_ArrFuncs & funcs = *funcs_ptr;
@@ -63,11 +84,18 @@ namespace eigenpy
     funcs.copyswap = copyswap;
     funcs.copyswapn = copyswapn;
     funcs.dotfunc = dotfunc;
+    funcs.fill = fill;
+    funcs.fillwithscalar = fillwithscalar;
     //      f->cast = cast;
     
     const int code = call_PyArray_RegisterDataType(descr_ptr);
     assert(code >= 0 && "The return code should be positive");
     PyArray_Descr * new_descr = call_PyArray_DescrFromType(code);
+
+    if(PyDict_SetItemString(py_type_ptr->tp_dict,"dtype",(PyObject*)descr_ptr) < 0)
+    {
+      throw std::invalid_argument("PyDict_SetItemString fails.");
+    }
     
     instance().type_to_py_type_bindings.insert(std::make_pair(type_info_ptr,py_type_ptr));
     instance().py_array_descr_bindings[py_type_ptr] = new_descr;

unittest/python/test_user_type.py

 import user_type
+import numpy as np
+#from packaging import version
 
 rows = 10
 cols = 20
 
-def test(mat):
-  mat.fill(mat.dtype.type(10.))
+def test(dtype):
+  mat = np.ones((rows,cols),dtype=dtype)
   mat_copy = mat.copy()
   assert (mat == mat_copy).all()
   assert not (mat != mat_copy).all()
 
+#  if version.parse(np.__version__) >= version.parse("1.21.0"): # check if it fixes for new versio of NumPy 
+#    mat.fill(mat.dtype.type(20.))
+#    mat_copy = mat.copy()
+#    assert((mat == mat_copy).all())
+#    assert(not (mat != mat_copy).all())
+
   mat_op = mat + mat
   mat_op = mat.copy(order='F') + mat.copy(order='C')
   
@@ -17,15 +25,32 @@ def test(mat):
   mat_op = mat.dot(mat.T)
   mat_op = mat / mat
 
-  mat_op = -mat;
+  mat_op = -mat
 
   assert (mat >= mat).all()
   assert (mat <= mat).all()
   assert not (mat > mat).all()
   assert not (mat < mat).all()
 
-mat = user_type.create_double(rows,cols)
-test(mat)
+def test_cast(from_dtype,to_dtype):
+  np.can_cast(from_dtype,to_dtype)
+
+  from_mat = np.zeros((rows,cols),dtype=from_dtype)
+  to_mat = from_mat.astype(dtype=to_dtype)
+  
+test(user_type.CustomDouble)
+
+test_cast(user_type.CustomDouble,np.double)
+test_cast(np.double,user_type.CustomDouble)
+
+test_cast(user_type.CustomDouble,np.int64)
+test_cast(np.int64,user_type.CustomDouble)
+
+test_cast(user_type.CustomDouble,np.int32)
+test_cast(np.int32,user_type.CustomDouble)
+
+test(user_type.CustomFloat)
 
-mat = user_type.create_float(rows,cols)
-test(mat)
+v = user_type.CustomDouble(1)
+a = np.array(v)
+assert type(v) == a.dtype.type

unittest/user_type.cpp

@@ -38,7 +38,7 @@ namespace Eigen
       MulCost               = 2
     };
     
-    static Scalar epsilon()
+    static CustomType<Scalar> epsilon()
     {
       return CustomType<Scalar>(std::numeric_limits<Scalar>::epsilon());
     }
@@ -97,6 +97,11 @@ struct CustomType
   
   CustomType operator-() const { return CustomType(-m_value); }
   
+  operator Scalar () const
+  {
+    return m_value;
+  }
+  
   std::string print() const
   {
     std::stringstream ss;
@@ -110,7 +115,7 @@ struct CustomType
     return os;
   }
  
-protected:
+//protected:
   
   Scalar m_value;
 };
@@ -186,4 +191,16 @@ BOOST_PYTHON_MODULE(user_type)
   bp::def("build_matrix",build_matrix<double>);
   bp::def("print",print<double>);
   bp::def("print",print<float>);
+  
+  eigenpy::registerCast<DoubleType,double>(true);
+  eigenpy::registerCast<double,DoubleType>(true);
+  eigenpy::registerCast<DoubleType,int32_t>(false);
+  eigenpy::registerCast<int32_t,DoubleType>(true);
+  eigenpy::registerCast<DoubleType,int64_t>(false);
+  eigenpy::registerCast<int64_t,DoubleType>(true);
+  eigenpy::registerCast<FloatType,int64_t>(false);
+  eigenpy::registerCast<int64_t,FloatType>(true);
+
+  bp::implicitly_convertible<double,DoubleType>();
+  bp::implicitly_convertible<DoubleType,double>();
 }