LCOV - code coverage report
Current view: top level - sql/backends/monet5/UDF/pyapi3 - conversion3.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 450 685 65.7 %
Date: 2020-06-29 20:00:14 Functions: 16 16 100.0 %

          Line data    Source code
       1             : /*
       2             :  * This Source Code Form is subject to the terms of the Mozilla Public
       3             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       4             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       5             :  *
       6             :  * Copyright 1997 - July 2008 CWI, August 2008 - 2020 MonetDB B.V.
       7             :  */
       8             : 
       9             : #include "monetdb_config.h"
      10             : #include "conversion.h"
      11             : #include "convert_loops.h"
      12             : #include "pytypes.h"
      13             : #include "type_conversion.h"
      14             : #include "unicode.h"
      15             : #include "blob.h"
      16             : #ifndef HAVE_EMBEDDED
      17             : #include "gdk_interprocess.h"
      18             : #endif
      19             : 
      20             : CREATE_SQL_FUNCTION_PTR(str, batbte_dec2_dbl);
      21             : CREATE_SQL_FUNCTION_PTR(str, batsht_dec2_dbl);
      22             : CREATE_SQL_FUNCTION_PTR(str, batint_dec2_dbl);
      23             : CREATE_SQL_FUNCTION_PTR(str, batlng_dec2_dbl);
      24             : #ifdef HAVE_HGE
      25             : CREATE_SQL_FUNCTION_PTR(str, bathge_dec2_dbl);
      26             : #endif
      27             : CREATE_SQL_FUNCTION_PTR(str, batstr_2time_timestamp);
      28             : CREATE_SQL_FUNCTION_PTR(str, batstr_2time_daytime);
      29             : CREATE_SQL_FUNCTION_PTR(str, batstr_2_date);
      30             : CREATE_SQL_FUNCTION_PTR(str, batdbl_num2dec_lng);
      31             : 
      32             : //! Wrapper to get eclass of SQL type
      33             : int GetSQLType(sql_subtype *sql_subtype);
      34             : 
      35         777 : static bool IsBlobType(int type)
      36             : {
      37         777 :         return type == TYPE_blob;
      38             : }
      39             : 
      40         306 : static bool IsVoidType(int type)
      41             : {
      42         306 :         return type == TYPE_void;
      43             : }
      44             : 
      45          91 : PyObject *PyArrayObject_FromScalar(PyInput *inp, char **return_message)
      46             : {
      47          91 :         PyObject *vararray = NULL;
      48          91 :         char *msg = NULL;
      49          91 :         assert(inp->scalar); // input has to be a scalar
      50             : 
      51          91 :         switch (inp->bat_type) {
      52           0 :                 case TYPE_void:
      53             : #if SIZEOF_OID == SIZEOF_INT
      54             :                         vararray = PyArray_Arange(0, 1, 1, NPY_UINT);
      55             : #else
      56           0 :                         vararray = PyArray_Arange(0, 1, 1, NPY_ULONGLONG);
      57             : #endif
      58           0 :                         break;
      59           0 :                 case TYPE_oid:
      60           0 :                         vararray = PyInt_FromLong((long)(*(oid *)inp->dataptr));
      61           0 :                         break;
      62           9 :                 case TYPE_bit:
      63           9 :                         vararray = PyInt_FromLong((long)(*(bit *)inp->dataptr));
      64           9 :                         break;
      65           0 :                 case TYPE_bte:
      66           0 :                         vararray = PyInt_FromLong((long)(*(bte *)inp->dataptr));
      67           0 :                         break;
      68           0 :                 case TYPE_sht:
      69           0 :                         vararray = PyInt_FromLong((long)(*(sht *)inp->dataptr));
      70           0 :                         break;
      71          63 :                 case TYPE_int:
      72          63 :                         vararray = PyInt_FromLong((long)(*(int *)inp->dataptr));
      73          63 :                         break;
      74           0 :                 case TYPE_lng:
      75           0 :                         vararray = PyLong_FromLongLong((*(lng *)inp->dataptr));
      76           0 :                         break;
      77           0 :                 case TYPE_flt:
      78           0 :                         vararray = PyFloat_FromDouble((double)(*(flt *)inp->dataptr));
      79           0 :                         break;
      80           1 :                 case TYPE_dbl:
      81           1 :                         vararray = PyFloat_FromDouble((double)(*(dbl *)inp->dataptr));
      82           1 :                         break;
      83             : #ifdef HAVE_HGE
      84           0 :                 case TYPE_hge:
      85           0 :                         vararray = PyLong_FromHge(*((hge *)inp->dataptr));
      86           0 :                         break;
      87             : #endif
      88          18 :                 case TYPE_str:
      89          18 :                         vararray = PyUnicode_FromString(*((char **)inp->dataptr));
      90          18 :                         break;
      91           0 :                 default:
      92           0 :                         msg = createException(MAL, "pyapi3.eval",
      93             :                                                                   SQLSTATE(PY000) "Unsupported scalar type %i.", inp->bat_type);
      94           0 :                         goto wrapup;
      95             :         }
      96          91 :         if (vararray == NULL) {
      97           0 :                 msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Something went wrong "
      98             :                                                                                                  "converting the MonetDB "
      99             :                                                                                                  "scalar to a Python scalar.");
     100           0 :                 goto wrapup;
     101             :         }
     102          91 : wrapup:
     103          91 :         *return_message = msg;
     104          91 :         return vararray;
     105             : }
     106             : 
     107         306 : PyObject *PyMaskedArray_FromBAT(PyInput *inp, size_t t_start, size_t t_end,
     108             :                                                                 char **return_message, bool copy)
     109             : {
     110         306 :         BAT *b;
     111         306 :         char *msg;
     112         306 :         PyObject *vararray;
     113             : 
     114         306 :         vararray = PyArrayObject_FromBAT(inp, t_start, t_end, return_message, copy);
     115         306 :         if (vararray == NULL) {
     116             :                 return NULL;
     117             :         }
     118         306 :         b = inp->bat;
     119             :         // To deal with null values, we use the numpy masked array structure
     120             :         // The masked array structure is an object with two arrays of equal size, a
     121             :         // data array and a mask array
     122             :         // The mask array is a boolean array that has the value 'True' when the
     123             :         // element is NULL, and 'False' otherwise
     124             :         // if we know for sure that the BAT has no NULL values, we can skip the construction
     125             :         // of this masked array. Otherwise, we create it.
     126         306 :         if (b->tnil || !b->tnonil) {
     127          32 :                 PyObject *mask;
     128          32 :                 PyObject *mafunc = PyObject_GetAttrString(
     129             :                         PyImport_Import(PyString_FromString("numpy.ma")), "masked_array");
     130          32 :                 PyObject *maargs;
     131          32 :                 PyObject *nullmask = PyNullMask_FromBAT(b, t_start, t_end);
     132             : 
     133          32 :                 if (!nullmask) {
     134           0 :                         msg = createException(MAL, "pyapi3.eval", "Failed to create mask for some reason");
     135           0 :                         goto wrapup;
     136          32 :                 } else if (nullmask == Py_None) {
     137           8 :                         maargs = PyTuple_New(1);
     138           8 :                         PyTuple_SetItem(maargs, 0, vararray);
     139             :                 } else {
     140          24 :                         maargs = PyTuple_New(2);
     141          24 :                         PyTuple_SetItem(maargs, 0, vararray);
     142          24 :                         PyTuple_SetItem(maargs, 1, (PyObject *)nullmask);
     143             :                 }
     144             : 
     145             :                 // Now we will actually construct the mask by calling the masked array
     146             :                 // constructor
     147          32 :                 mask = PyObject_CallObject(mafunc, maargs);
     148          32 :                 if (!mask) {
     149           0 :                         msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Failed to create mask");
     150           0 :                         goto wrapup;
     151             :                 }
     152          32 :                 Py_DECREF(maargs);
     153          32 :                 Py_DECREF(mafunc);
     154             : 
     155             :                 vararray = mask;
     156             :         }
     157             :         return vararray;
     158           0 : wrapup:
     159           0 :         *return_message = msg;
     160           0 :         return NULL;
     161             : }
     162             : 
     163         306 : PyObject *PyArrayObject_FromBAT(PyInput *inp, size_t t_start, size_t t_end,
     164             :                                                                 char **return_message, bool copy)
     165             : {
     166             :         // This variable will hold the converted Python object
     167         306 :         PyObject *vararray = NULL;
     168         306 :         char *msg;
     169         306 :         size_t j = 0;
     170         306 :         BUN p = 0, q = 0;
     171         306 :         BATiter li;
     172         306 :         BAT *b = inp->bat;
     173         306 :         npy_intp elements[1] = {t_end - t_start};
     174             : 
     175         306 :         assert(!inp->scalar); // input has to be a BAT
     176             : 
     177         306 :         if (!b) {
     178             :                 // No BAT was found, we can't do anything in this case
     179           0 :                 msg = createException(MAL, "pyapi3.eval", SQLSTATE(HY013) MAL_MALLOC_FAIL " bat missing");
     180           0 :                 goto wrapup;
     181             :         }
     182             : 
     183         603 :         if (!IsVoidType(inp->bat_type) && !IsBlobType(inp->bat_type) &&
     184         589 :                 (!IsStandardBATType(inp->bat_type) ||
     185         292 :                  ConvertableSQLType(inp->sql_subtype))) { // if the sql type is set, we
     186             :                                                                                                   // have to do some conversion
     187           7 :                 if (inp->scalar) {
     188             :                         // FIXME: scalar SQL types
     189           0 :                         msg = createException(
     190             :                                 MAL, "pyapi3.eval",
     191             :                                 SQLSTATE(0A000) "Scalar SQL types haven't been implemented yet... sorry");
     192           0 :                         goto wrapup;
     193             :                 } else {
     194           7 :                         BAT *ret_bat = NULL;
     195           7 :                         msg = ConvertFromSQLType(inp->bat, inp->sql_subtype, &ret_bat,
     196             :                                                                          &inp->bat_type);
     197           7 :                         if (msg != MAL_SUCCEED) {
     198           0 :                                 freeException(msg);
     199           0 :                                 msg = createException(MAL, "pyapi3.eval",
     200             :                                                                           SQLSTATE(PY000) "Failed to convert BAT.");
     201           0 :                                 goto wrapup;
     202             :                         }
     203           7 :                         b = ret_bat;
     204             :                 }
     205             :         }
     206             : 
     207         306 :         if (IsBlobType(inp->bat_type)) {
     208           2 :                 PyObject **data;
     209           2 :                 li = bat_iterator(b);
     210           2 :                 vararray = PyArray_EMPTY(1, elements, NPY_OBJECT, 0);
     211           2 :                 data = PyArray_DATA((PyArrayObject *)vararray);
     212           8 :                 BATloop(b, p, q)
     213             :                 {
     214           6 :                         blob *t = (blob *)BUNtvar(li, p);
     215           6 :                         if (t->nitems == ~(size_t)0) {
     216           0 :                                 data[p] = Py_None;
     217           0 :                                 Py_INCREF(Py_None);
     218             :                         } else {
     219           6 :                                 data[p] = PyByteArray_FromStringAndSize(t->data, t->nitems);
     220             :                         }
     221             :                 }
     222             :         } else {
     223         304 :                 switch (inp->bat_type) {
     224           7 :                         case TYPE_void:
     225             : #if SIZEOF_OID == SIZEOF_INT
     226             :                                 BAT_TO_NP_CREATE_ALWAYS(b, NPY_UINT);
     227             : #else
     228           7 :                                 BAT_TO_NP_CREATE_ALWAYS(b, NPY_ULONGLONG);
     229             : #endif
     230           7 :                                 break;
     231          11 :                         case TYPE_oid:
     232             : #if SIZEOF_OID == SIZEOF_INT
     233             :                                 BAT_TO_NP(b, oid, NPY_UINT32);
     234             : #else
     235          11 :                                 BAT_TO_NP(b, oid, NPY_UINT64);
     236             : #endif
     237             :                                 break;
     238           1 :                         case TYPE_bit:
     239           1 :                                 BAT_TO_NP(b, bit, NPY_INT8);
     240             :                                 break;
     241           1 :                         case TYPE_bte:
     242           1 :                                 BAT_TO_NP(b, bte, NPY_INT8);
     243             :                                 break;
     244           0 :                         case TYPE_sht:
     245           0 :                                 BAT_TO_NP(b, sht, NPY_INT16);
     246             :                                 break;
     247         263 :                         case TYPE_int:
     248         263 :                                 BAT_TO_NP(b, int, NPY_INT32);
     249             :                                 break;
     250           0 :                         case TYPE_lng:
     251           0 :                                 BAT_TO_NP(b, lng, NPY_INT64);
     252             :                                 break;
     253           0 :                         case TYPE_flt:
     254           0 :                                 BAT_TO_NP(b, flt, NPY_FLOAT32);
     255             :                                 break;
     256          10 :                         case TYPE_dbl:
     257          10 :                                 BAT_TO_NP(b, dbl, NPY_FLOAT64);
     258             :                                 break;
     259          11 :                         case TYPE_str: {
     260          11 :                                 bool unicode = false;
     261          11 :                                 li = bat_iterator(b);
     262             :                                 // create a NPY_OBJECT array object
     263          11 :                                 vararray = PyArray_New(&PyArray_Type, 1, elements, NPY_OBJECT,
     264             :                                                                            NULL, NULL, 0, 0, NULL);
     265             : 
     266     1000040 :                                 BATloop(b, p, q)
     267             :                                 {
     268     1000030 :                                         char *t = (char *)BUNtvar(li, p);
     269     6000200 :                                         for (; *t != 0; t++) {
     270     5000170 :                                                 if (*t & 0x80) {
     271             :                                                         unicode = true;
     272             :                                                         break;
     273             :                                                 }
     274             :                                         }
     275     1000030 :                                         if (unicode) {
     276             :                                                 break;
     277             :                                         }
     278             :                                 }
     279             : 
     280             :                                 {
     281          11 :                                         PyObject **data =
     282          11 :                                                 ((PyObject **)PyArray_DATA((PyArrayObject *)vararray));
     283          11 :                                         PyObject *obj;
     284          11 :                                         j = 0;
     285          11 :                                         if (unicode) {
     286           1 :                                                 if (GDK_ELIMDOUBLES(b->tvheap)) {
     287           2 :                                                         PyObject **pyptrs =
     288           1 :                                                                 GDKzalloc(b->tvheap->free * sizeof(PyObject *));
     289           1 :                                                         if (!pyptrs) {
     290           0 :                                                                 msg = createException(MAL, "pyapi3.eval",
     291             :                                                                                                           SQLSTATE(HY013) MAL_MALLOC_FAIL
     292             :                                                                                                           " PyObject strings.");
     293           0 :                                                                 goto wrapup;
     294             :                                                         }
     295           3 :                                                         BATloop(b, p, q)
     296             :                                                         {
     297           2 :                                                                 const char *t = (const char *)BUNtvar(li, p);
     298           2 :                                                                 ptrdiff_t offset = t - b->tvheap->base;
     299           2 :                                                                 if (!pyptrs[offset]) {
     300           4 :                                                                         if (strNil(t)) {
     301             :                                                                                 // str_nil isn't a valid UTF-8 character
     302             :                                                                                 // (it's 0x80), so we can't decode it as
     303             :                                                                                 // UTF-8 (it will throw an error)
     304           1 :                                                                                 pyptrs[offset] =
     305           1 :                                                                                         PyUnicode_FromString("-");
     306             :                                                                         } else {
     307             :                                                                                 // otherwise we can just decode the
     308             :                                                                                 // string as UTF-8
     309           1 :                                                                                 pyptrs[offset] =
     310           1 :                                                                                         PyUnicode_FromString(t);
     311             :                                                                         }
     312           2 :                                                                         if (!pyptrs[offset]) {
     313           0 :                                                                                 msg = createException(
     314             :                                                                                         MAL, "pyapi3.eval",
     315             :                                                                                         SQLSTATE(PY000) "Failed to create string.");
     316           0 :                                                                                 goto wrapup;
     317             :                                                                         }
     318             :                                                                 } else {
     319           0 :                                                                         Py_INCREF(pyptrs[offset]);
     320             :                                                                 }
     321           2 :                                                                 data[j++] = pyptrs[offset];
     322             :                                                         }
     323           1 :                                                         GDKfree(pyptrs);
     324             :                                                 } else {
     325           0 :                                                         BATloop(b, p, q)
     326             :                                                         {
     327           0 :                                                                 char *t = (char *)BUNtvar(li, p);
     328           0 :                                                                 if (strNil(t)) {
     329             :                                                                         // str_nil isn't a valid UTF-8 character
     330             :                                                                         // (it's 0x80), so we can't decode it as
     331             :                                                                         // UTF-8 (it will throw an error)
     332           0 :                                                                         obj = PyUnicode_FromString("-");
     333             :                                                                 } else {
     334             :                                                                         // otherwise we can just decode the string
     335             :                                                                         // as UTF-8
     336           0 :                                                                         obj = PyUnicode_FromString(t);
     337             :                                                                 }
     338             : 
     339           0 :                                                                 if (obj == NULL) {
     340           0 :                                                                         msg = createException(
     341             :                                                                                 MAL, "pyapi3.eval",
     342             :                                                                                 SQLSTATE(PY000) "Failed to create string.");
     343           0 :                                                                         goto wrapup;
     344             :                                                                 }
     345           0 :                                                                 data[j++] = obj;
     346             :                                                         }
     347             :                                                 }
     348             :                                         } else {
     349             :                                                 /* special case where we exploit the
     350             :                                                  * duplicate-eliminated string heap */
     351          10 :                                                 if (GDK_ELIMDOUBLES(b->tvheap)) {
     352          20 :                                                         PyObject **pyptrs =
     353          10 :                                                                 GDKzalloc(b->tvheap->free * sizeof(PyObject *));
     354          10 :                                                         if (!pyptrs) {
     355           0 :                                                                 msg = createException(MAL, "pyapi3.eval",
     356             :                                                                                                           SQLSTATE(HY013) MAL_MALLOC_FAIL
     357             :                                                                                                           " PyObject strings.");
     358           0 :                                                                 goto wrapup;
     359             :                                                         }
     360     1000040 :                                                         BATloop(b, p, q)
     361             :                                                         {
     362     1000030 :                                                                 const char *t = (const char *)BUNtvar(li, p);
     363     1000030 :                                                                 ptrdiff_t offset = t - b->tvheap->base;
     364     1000030 :                                                                 if (!pyptrs[offset]) {
     365          21 :                                                                         pyptrs[offset] = PyString_FromString(t);
     366             :                                                                 } else {
     367     1000010 :                                                                         Py_INCREF(pyptrs[offset]);
     368             :                                                                 }
     369     1000030 :                                                                 data[j++] = pyptrs[offset];
     370             :                                                         }
     371          10 :                                                         GDKfree(pyptrs);
     372             :                                                 } else {
     373           0 :                                                         BATloop(b, p, q)
     374             :                                                         {
     375           0 :                                                                 char *t = (char *)BUNtvar(li, p);
     376           0 :                                                                 obj = PyString_FromString(t);
     377           0 :                                                                 if (obj == NULL) {
     378           0 :                                                                         msg = createException(
     379             :                                                                                 MAL, "pyapi3.eval",
     380             :                                                                                 SQLSTATE(PY000) "Failed to create string.");
     381           0 :                                                                         goto wrapup;
     382             :                                                                 }
     383           0 :                                                                 data[j++] = obj;
     384             :                                                         }
     385             :                                                 }
     386             :                                         }
     387             :                                 }
     388             :                         } break;
     389             : #ifdef HAVE_HGE
     390           0 :                         case TYPE_hge: {
     391             :                                 // create a NPY_FLOAT64 array to hold the huge type
     392           0 :                                 vararray = PyArray_New(&PyArray_Type, 1,
     393           0 :                                                                            (npy_intp[1]){t_end - t_start},
     394             :                                                                            NPY_FLOAT64, NULL, NULL, 0, 0, NULL);
     395             : 
     396           0 :                                 j = 0;
     397           0 :                                 npy_float64 *data = (npy_float64 *)PyArray_DATA((PyArrayObject *)vararray);
     398           0 :                                 const hge *vals = (const hge *) Tloc(b, 0);
     399           0 :                                 BATloop(b, p, q) {
     400           0 :                                         data[j++] = (npy_float64)vals[p];
     401             :                                 }
     402           0 :                                 break;
     403             :                         }
     404             : #endif
     405           0 :                         default:
     406           0 :                                 if (!inp->sql_subtype || !inp->sql_subtype->type) {
     407           0 :                                         msg = createException(MAL, "pyapi3.eval",
     408             :                                                                                   SQLSTATE(PY000) "unknown argument type");
     409             :                                 } else {
     410           0 :                                         msg = createException(MAL, "pyapi3.eval",
     411             :                                                                                   SQLSTATE(PY000) "Unsupported SQL Type: %s",
     412             :                                                                                   inp->sql_subtype->type->sqlname);
     413             :                                 }
     414           0 :                                 goto wrapup;
     415             :                 }
     416             :         }
     417             : 
     418         306 :         if (vararray == NULL) {
     419           0 :                 msg = createException(MAL, "pyapi3.eval",
     420             :                                                           SQLSTATE(PY000) "Failed to convert BAT to Numpy array.");
     421           0 :                 goto wrapup;
     422             :         }
     423         306 :         if (b != inp->bat)
     424           7 :                 BBPunfix(b->batCacheid);
     425             :         return vararray;
     426           0 : wrapup:
     427           0 :         *return_message = msg;
     428           0 :         if (b != inp->bat)
     429           0 :                 BBPunfix(b->batCacheid);
     430             :         return NULL;
     431             : }
     432             : 
     433             : #define CreateNullMask(tpe)                                                    \
     434             :         {                                                                          \
     435             :                 tpe *bat_ptr = (tpe *)b->theap.base;                                   \
     436             :                 for (j = 0; j < count; j++) {                                          \
     437             :                         mask_data[j] = is_##tpe##_nil(bat_ptr[j]);                         \
     438             :                         found_nil |= mask_data[j];                                         \
     439             :                 }                                                                      \
     440             :         }
     441             : 
     442          32 : PyObject *PyNullMask_FromBAT(BAT *b, size_t t_start, size_t t_end)
     443             : {
     444             :         // We will now construct the Masked array, we start by setting everything to
     445             :         // False
     446          32 :         size_t count = t_end - t_start;
     447          32 :         npy_intp elements[1] = {count};
     448          32 :         PyArrayObject *nullmask =
     449          32 :                 (PyArrayObject *)PyArray_EMPTY(1, elements, NPY_BOOL, 0);
     450          32 :         const void *nil = ATOMnilptr(b->ttype);
     451          32 :         size_t j;
     452          32 :         bool found_nil = false;
     453          32 :         BATiter bi = bat_iterator(b);
     454          32 :         bool *mask_data = (bool *)PyArray_DATA(nullmask);
     455             : 
     456          63 :         switch (ATOMbasetype(getBatType(b->ttype))) {
     457           0 :                 case TYPE_bit:
     458           0 :                         CreateNullMask(bit);
     459             :                         break;
     460           0 :                 case TYPE_bte:
     461           0 :                         CreateNullMask(bte);
     462             :                         break;
     463           0 :                 case TYPE_sht:
     464           0 :                         CreateNullMask(sht);
     465             :                         break;
     466          29 :                 case TYPE_int:
     467      135076 :                         CreateNullMask(int);
     468             :                         break;
     469           1 :                 case TYPE_lng:
     470           3 :                         CreateNullMask(lng);
     471             :                         break;
     472           0 :                 case TYPE_flt:
     473           0 :                         CreateNullMask(flt);
     474             :                         break;
     475           2 :                 case TYPE_dbl:
     476          16 :                         CreateNullMask(dbl);
     477             :                         break;
     478             : #ifdef HAVE_HGE
     479           0 :                 case TYPE_hge:
     480           0 :                         CreateNullMask(hge);
     481             :                         break;
     482             : #endif
     483           0 :                 default: {
     484           0 :                         int (*atomcmp)(const void *, const void *) = ATOMcompare(b->ttype);
     485           0 :                         for (j = 0; j < count; j++) {
     486           0 :                                 mask_data[j] = (*atomcmp)(BUNtail(bi, (BUN)(j)), nil) == 0;
     487           0 :                                 found_nil |= mask_data[j];
     488             :                         }
     489             :                 }
     490             :         }
     491             : 
     492          32 :         if (!found_nil) {
     493           8 :                 Py_DECREF(nullmask);
     494           8 :                 Py_RETURN_NONE;
     495             :         }
     496             : 
     497             :         return (PyObject *)nullmask;
     498             : }
     499             : 
     500          35 : PyObject *PyDict_CheckForConversion(PyObject *pResult, int expected_columns,
     501             :                                                                         char **retcol_names, char **return_message)
     502             : {
     503          35 :         char *msg = MAL_SUCCEED;
     504          35 :         PyObject *result = PyList_New(expected_columns),
     505          35 :                          *keys = PyDict_Keys(pResult);
     506          35 :         int i;
     507             : 
     508         150 :         for (i = 0; i < expected_columns; i++) {
     509          83 :                 PyObject *object = PyDict_GetItemString(pResult, retcol_names[i]);
     510          83 :                 if (object == NULL) {
     511           2 :                         msg =
     512           2 :                                 createException(MAL, "pyapi3.eval",
     513             :                                                                 SQLSTATE(PY000) "Expected a return value with name \"%s\", but "
     514             :                                                                 "this key was not present in the dictionary.",
     515             :                                                                 retcol_names[i]);
     516           2 :                         goto wrapup;
     517             :                 }
     518          81 :                 Py_INCREF(object);
     519          81 :                 object = PyObject_CheckForConversion(object, 1, NULL, return_message);
     520          81 :                 if (object == NULL) {
     521           1 :                         msg = createException(
     522             :                                 MAL, "pyapi3.eval",
     523             :                                 SQLSTATE(PY000) "Error converting dict return value \"%s\": %s.",
     524             :                                 retcol_names[i], getExceptionMessage(*return_message));
     525           1 :                         GDKfree(*return_message);
     526           1 :                         goto wrapup;
     527             :                 }
     528          80 :                 if (PyList_CheckExact(object)) {
     529          80 :                         PyObject *item = PyList_GetItem(object, 0);
     530          80 :                         PyList_SetItem(result, i, item);
     531          80 :                         Py_INCREF(item);
     532          80 :                         Py_DECREF(object);
     533             :                 } else {
     534           0 :                         msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Why is this not a list?");
     535           0 :                         goto wrapup;
     536             :                 }
     537             :         }
     538          32 :         Py_DECREF(keys);
     539          32 :         Py_DECREF(pResult);
     540             :         // Py_INCREF(result);
     541             :         return result;
     542           3 : wrapup:
     543           3 :         *return_message = msg;
     544           3 :         Py_DECREF(result);
     545           3 :         Py_DECREF(keys);
     546           3 :         Py_DECREF(pResult);
     547             :         return NULL;
     548             : }
     549             : 
     550         173 : PyObject *PyObject_CheckForConversion(PyObject *pResult, int expected_columns,
     551             :                                                                           int *actual_columns,
     552             :                                                                           char **return_message)
     553             : {
     554         173 :         char *msg;
     555         173 :         int columns = 0;
     556         173 :         if (pResult) {
     557         173 :                 PyObject *pColO = NULL;
     558         173 :                 if (PyType_IsPandasDataFrame(pResult)) {
     559             :                         // the result object is a Pandas data frame
     560             :                         // we can convert the pandas data frame to a numpy array by simply
     561             :                         // accessing the "values" field (as pandas dataframes are numpy
     562             :                         // arrays internally)
     563           0 :                         pResult = PyObject_GetAttrString(pResult, "values");
     564           0 :                         if (pResult == NULL) {
     565           0 :                                 msg = createException(MAL, "pyapi3.eval",
     566             :                                                                           SQLSTATE(PY000) "Invalid Pandas data frame.");
     567           0 :                                 goto wrapup;
     568             :                         }
     569             :                         // we transpose the values field so it's aligned correctly for our
     570             :                         // purposes
     571           0 :                         pResult = PyObject_GetAttrString(pResult, "T");
     572           0 :                         if (pResult == NULL) {
     573           0 :                                 msg = createException(MAL, "pyapi3.eval",
     574             :                                                                           SQLSTATE(PY000) "Invalid Pandas data frame.");
     575           0 :                                 goto wrapup;
     576             :                         }
     577             :                 }
     578             : 
     579         173 :                 if (PyType_IsPyScalar(
     580             :                                 pResult)) { // check if the return object is a scalar
     581          56 :                         if (expected_columns == 1 || expected_columns <= 0) {
     582             :                                 // if we only expect a single return value, we can accept
     583             :                                 // scalars by converting it into an array holding an array
     584             :                                 // holding the element (i.e. [[pResult]])
     585          56 :                                 PyObject *list = PyList_New(1);
     586          56 :                                 PyList_SetItem(list, 0, pResult);
     587          56 :                                 pResult = list;
     588             : 
     589          56 :                                 list = PyList_New(1);
     590          56 :                                 PyList_SetItem(list, 0, pResult);
     591          56 :                                 pResult = list;
     592             : 
     593          56 :                                 columns = 1;
     594             :                         } else {
     595             :                                 // the result object is a scalar, yet we expect more than one
     596             :                                 // return value. We can only convert the result into a list with
     597             :                                 // a single element, so the output is necessarily wrong.
     598           0 :                                 msg = createException(
     599             :                                         MAL, "pyapi3.eval",
     600             :                                         SQLSTATE(PY000) "A single scalar was returned, yet we expect a list of %d "
     601             :                                         "columns. We can only convert a single scalar into a "
     602             :                                         "single column, thus the result is invalid.",
     603             :                                         expected_columns);
     604           0 :                                 goto wrapup;
     605             :                         }
     606             :                 } else {
     607             :                         // if it is not a scalar, we check if it is a single array
     608         117 :                         bool IsSingleArray = TRUE;
     609         117 :                         PyObject *data = pResult;
     610         117 :                         if (PyType_IsNumpyMaskedArray(data)) {
     611          31 :                                 data = PyObject_GetAttrString(pResult, "data");
     612          31 :                                 if (data == NULL) {
     613           0 :                                         msg = createException(MAL, "pyapi3.eval",
     614             :                                                                                   SQLSTATE(PY000) "Invalid masked array.");
     615           0 :                                         goto wrapup;
     616             :                                 }
     617             :                         }
     618         117 :                         if (PyType_IsNumpyArray(data)) {
     619          84 :                                 if (PyArray_NDIM((PyArrayObject *)data) != 1) {
     620             :                                         IsSingleArray = FALSE;
     621             :                                 } else {
     622          84 :                                         pColO = PyArray_GETITEM(
     623             :                                                 (PyArrayObject *)data,
     624          84 :                                                 PyArray_GETPTR1((PyArrayObject *)data, 0));
     625          84 :                                         IsSingleArray = PyType_IsPyScalar(pColO);
     626             :                                 }
     627          33 :                         } else if (PyList_Check(data)) {
     628          33 :                                 pColO = PyList_GetItem(data, 0);
     629          33 :                                 IsSingleArray = PyType_IsPyScalar(pColO);
     630           0 :                         } else if (!PyType_IsNumpyMaskedArray(data)) {
     631             :                                 // it is neither a python array, numpy array or numpy masked
     632             :                                 // array, thus the result is unsupported! Throw an exception!
     633           0 :                                 msg = createException(
     634             :                                         MAL, "pyapi3.eval",
     635             :                                         SQLSTATE(PY000) "Unsupported result object. Expected either a list, "
     636             :                                         "dictionary, a numpy array, a numpy masked array or a "
     637             :                                         "pandas data frame, but received an object of type \"%s\"",
     638             :                                         PyString_AsString(PyObject_Str(PyObject_Type(data))));
     639           0 :                                 goto wrapup;
     640             :                         }
     641             : 
     642         117 :                         if (IsSingleArray) {
     643         110 :                                 if (expected_columns == 1 || expected_columns <= 0) {
     644             :                                         // if we only expect a single return value, we can accept a
     645             :                                         // single array by converting it into an array holding an
     646             :                                         // array holding the element (i.e. [pResult])
     647         110 :                                         PyObject *list = PyList_New(1);
     648         110 :                                         PyList_SetItem(list, 0, pResult);
     649         110 :                                         pResult = list;
     650             : 
     651         110 :                                         columns = 1;
     652             :                                 } else {
     653             :                                         // the result object is a single array, yet we expect more
     654             :                                         // than one return value. We can only convert the result
     655             :                                         // into a list with a single array, so the output is
     656             :                                         // necessarily wrong.
     657           0 :                                         msg = createException(MAL, "pyapi3.eval",
     658             :                                                                                   SQLSTATE(PY000) "A single array was returned, yet we "
     659             :                                                                                   "expect a list of %d columns. The "
     660             :                                                                                   "result is invalid.",
     661             :                                                                                   expected_columns);
     662           0 :                                         goto wrapup;
     663             :                                 }
     664             :                         } else {
     665             :                                 // the return value is an array of arrays, all we need to do is
     666             :                                 // check if it is the correct size
     667           7 :                                 int results = 0;
     668           7 :                                 if (PyList_Check(data))
     669           7 :                                         results = (int)PyList_Size(data);
     670             :                                 else
     671           0 :                                         results = (int)PyArray_DIMS((PyArrayObject *)data)[0];
     672           7 :                                 columns = results;
     673           7 :                                 if (results != expected_columns && expected_columns > 0) {
     674             :                                         // wrong return size, we expect pci->retc arrays
     675           1 :                                         msg = createException(MAL, "pyapi3.eval",
     676             :                                                                                   SQLSTATE(PY000) "An array of size %d was returned, "
     677             :                                                                                   "yet we expect a list of %d columns. "
     678             :                                                                                   "The result is invalid.",
     679             :                                                                                   results, expected_columns);
     680           1 :                                         goto wrapup;
     681             :                                 }
     682             :                         }
     683             :                 }
     684             :         } else {
     685           0 :                 msg = createException(
     686             :                         MAL, "pyapi3.eval",
     687             :                         SQLSTATE(PY000) "Invalid result object. No result object could be generated.");
     688           0 :                 goto wrapup;
     689             :         }
     690             : 
     691         172 :         if (actual_columns != NULL)
     692           0 :                 *actual_columns = columns;
     693             :         return pResult;
     694           1 : wrapup:
     695           1 :         if (actual_columns != NULL)
     696           0 :                 *actual_columns = columns;
     697           1 :         *return_message = msg;
     698           1 :         return NULL;
     699             : }
     700             : 
     701         213 : str PyObject_GetReturnValues(PyObject *obj, PyReturn *ret)
     702             : {
     703         213 :         PyObject *pMask = NULL;
     704         213 :         str msg = MAL_SUCCEED;
     705             :         // If it isn't we need to convert pColO to the expected Numpy Array type
     706         213 :         ret->numpy_array = PyArray_FromAny(
     707             :                 obj, NULL, 1, 1, NPY_ARRAY_CARRAY | NPY_ARRAY_FORCECAST, NULL);
     708         213 :         if (ret->numpy_array == NULL) {
     709           0 :                 msg = createException(
     710             :                         MAL, "pyapi3.eval",
     711             :                         SQLSTATE(PY000) "Could not create a Numpy array from the return type.\n");
     712           0 :                 goto wrapup;
     713             :         }
     714             : 
     715         213 :         ret->result_type =
     716         213 :                 PyArray_DESCR((PyArrayObject *)ret->numpy_array)
     717         213 :                         ->type_num; // We read the result type from the resulting array
     718         213 :         ret->memory_size = PyArray_DESCR((PyArrayObject *)ret->numpy_array)->elsize;
     719         213 :         ret->count = PyArray_DIMS((PyArrayObject *)ret->numpy_array)[0];
     720         213 :         ret->array_data = PyArray_DATA((PyArrayObject *)ret->numpy_array);
     721         213 :         ret->mask_data = NULL;
     722         213 :         ret->numpy_mask = NULL;
     723             :         // If pColO is a Masked array, we convert the mask to a NPY_BOOL numpy array
     724         213 :         if (PyObject_HasAttrString(obj, "mask")) {
     725          31 :                 pMask = PyObject_GetAttrString(obj, "mask");
     726          31 :                 if (pMask != NULL) {
     727          62 :                         ret->numpy_mask =
     728          31 :                                 PyArray_FromAny(pMask, PyArray_DescrFromType(NPY_BOOL), 1, 1,
     729             :                                                                 NPY_ARRAY_CARRAY, NULL);
     730          31 :                         if (ret->numpy_mask == NULL ||
     731          30 :                                 PyArray_DIMS((PyArrayObject *)ret->numpy_mask)[0] !=
     732          30 :                                         (int)ret->count) {
     733           1 :                                 PyErr_Clear();
     734           1 :                                 pMask = NULL;
     735           1 :                                 ret->numpy_mask = NULL;
     736             :                         }
     737             :                 }
     738             :         }
     739         213 :         if (ret->numpy_mask != NULL)
     740          30 :                 ret->mask_data = PyArray_DATA((PyArrayObject *)ret->numpy_mask);
     741         183 : wrapup:
     742         213 :         return msg;
     743             : }
     744             : 
     745         124 : bool PyObject_PreprocessObject(PyObject *pResult, PyReturn *pyreturn_values,
     746             :                                                            int column_count, char **return_message)
     747             : {
     748         124 :         int i;
     749         124 :         char *msg;
     750         296 :         for (i = 0; i < column_count; i++) {
     751             :                 // Refers to the current Numpy mask (if it exists)
     752         172 :                 PyObject *pMask = NULL;
     753             :                 // Refers to the current Numpy array
     754         172 :                 PyObject *pColO = NULL;
     755             :                 // This is the PyReturn header information for the current return value,
     756             :                 // we will fill this now
     757         172 :                 PyReturn *ret = &pyreturn_values[i];
     758             : 
     759         172 :                 ret->multidimensional = FALSE;
     760             :                 // There are three possibilities (we have ensured this right after
     761             :                 // executing the Python call by calling PyObject_CheckForConversion)
     762             :                 // 1: The top level result object is a PyList or Numpy Array containing
     763             :                 // pci->retc Numpy Arrays
     764             :                 // 2: The top level result object is a (pci->retc x N) dimensional Numpy
     765             :                 // Array [Multidimensional]
     766             :                 // 3: The top level result object is a (pci->retc x N) dimensional Numpy
     767             :                 // Masked Array [Multidimensional]
     768         172 :                 if (PyList_Check(pResult)) {
     769             :                         // If it is a PyList, we simply get the i'th Numpy array from the
     770             :                         // PyList
     771         172 :                         pColO = PyList_GetItem(pResult, i);
     772             :                 } else {
     773             :                         // If it isn't, the result object is either a Nump Masked Array or a
     774             :                         // Numpy Array
     775           0 :                         PyObject *data = pResult;
     776           0 :                         if (PyType_IsNumpyMaskedArray(data)) {
     777           0 :                                 data = PyObject_GetAttrString(
     778             :                                         pResult, "data"); // If it is a Masked array, the data is
     779             :                                                                           // stored in the masked_array.data
     780             :                                                                           // attribute
     781           0 :                                 pMask = PyObject_GetAttrString(pResult, "mask");
     782             :                         }
     783             : 
     784             :                         // We can either have a multidimensional numpy array, or a single
     785             :                         // dimensional numpy array
     786           0 :                         if (PyArray_NDIM((PyArrayObject *)data) != 1) {
     787             :                                 // If it is a multidimensional numpy array, we have to convert
     788             :                                 // the i'th dimension to a NUMPY array object
     789           0 :                                 ret->multidimensional = TRUE;
     790           0 :                                 ret->result_type =
     791           0 :                                         PyArray_DESCR((PyArrayObject *)data)->type_num;
     792             :                         } else {
     793             :                                 // If it is a single dimensional Numpy array, we get the i'th
     794             :                                 // Numpy array from the Numpy Array
     795           0 :                                 pColO =
     796           0 :                                         PyArray_GETITEM((PyArrayObject *)data,
     797           0 :                                                                         PyArray_GETPTR1((PyArrayObject *)data, i));
     798             :                         }
     799             :                 }
     800             : 
     801             :                 // Now we have to do some preprocessing on the data
     802         172 :                 if (ret->multidimensional) {
     803             :                         // If it is a multidimensional Numpy array, we don't need to do any
     804             :                         // conversion, we can just do some pointers
     805           0 :                         ret->count = PyArray_DIMS((PyArrayObject *)pResult)[1];
     806           0 :                         ret->numpy_array = pResult;
     807           0 :                         ret->numpy_mask = pMask;
     808           0 :                         ret->array_data = PyArray_DATA((PyArrayObject *)ret->numpy_array);
     809           0 :                         if (ret->numpy_mask != NULL)
     810           0 :                                 ret->mask_data = PyArray_DATA((PyArrayObject *)ret->numpy_mask);
     811           0 :                         ret->memory_size =
     812           0 :                                 PyArray_DESCR((PyArrayObject *)ret->numpy_array)->elsize;
     813             :                 } else {
     814         172 :                         msg = PyObject_GetReturnValues(pColO, ret);
     815         172 :                         if (msg != MAL_SUCCEED) {
     816           0 :                                 goto wrapup;
     817             :                         }
     818             :                 }
     819             :         }
     820             :         return TRUE;
     821           0 : wrapup:
     822           0 :         *return_message = msg;
     823           0 :         return FALSE;
     824             : }
     825             : 
     826         172 : BAT *PyObject_ConvertToBAT(PyReturn *ret, sql_subtype *type, int bat_type,
     827             :                                                    int i, oid seqbase, char **return_message, bool copy)
     828             : {
     829         172 :         BAT *b = NULL;
     830         172 :         size_t index_offset = 0;
     831         172 :         char *msg;
     832         172 :         size_t iu;
     833             : 
     834         172 :         if (ret->multidimensional)
     835           0 :                 index_offset = i;
     836             : 
     837         172 :         switch (GetSQLType(type)) {
     838           3 :                 case EC_TIMESTAMP:
     839             :                 case EC_TIME:
     840             :                 case EC_DATE:
     841           3 :                         bat_type = TYPE_str;
     842           3 :                         break;
     843           3 :                 case EC_DEC:
     844           3 :                         bat_type = TYPE_dbl;
     845           3 :                         break;
     846             :                 default:
     847             :                         break;
     848             :         }
     849             : 
     850         172 :         if (IsBlobType(bat_type)) {
     851           2 :                 bool *mask = NULL;
     852           2 :                 char *data = NULL;
     853           2 :                 blob *ele_blob;
     854           2 :                 size_t blob_fixed_size = ret->memory_size;
     855             : 
     856           2 :                 PyObject *pickle_module = NULL, *pickle = NULL;
     857           2 :                 bool gstate = 0;
     858             : 
     859           2 :                 if (ret->result_type == NPY_OBJECT) {
     860             :                         // Python objects, we may need to pickle them, so we
     861             :                         // may execute Python code, we have to obtain the GIL
     862           0 :                         gstate = Python_ObtainGIL();
     863           0 :                         pickle_module = PyImport_ImportModule("pickle");
     864           0 :                         if (pickle_module == NULL) {
     865           0 :                                 msg = createException(MAL, "pyapi3.eval",
     866             :                                                                           SQLSTATE(PY000) "Can't load pickle module to pickle python object to blob");
     867           0 :                                 Python_ReleaseGIL(gstate);
     868           0 :                                 goto wrapup;
     869             :                         }
     870             :                         blob_fixed_size = 0; // Size depends on the objects
     871             :                 }
     872             : 
     873           2 :                 if (ret->mask_data != NULL) {
     874             :                         mask = (bool *)ret->mask_data;
     875             :                 }
     876           2 :                 if (ret->array_data == NULL) {
     877           0 :                         msg = createException(MAL, "pyapi3.eval",
     878             :                                                                   SQLSTATE(PY000) "No return value stored in the structure.");
     879           0 :                         if (ret->result_type == NPY_OBJECT) {
     880           0 :                                 Py_XDECREF(pickle_module);
     881           0 :                                 Python_ReleaseGIL(gstate);
     882             :                         }
     883           0 :                         goto wrapup;
     884             :                 }
     885           2 :                 data = (char *)ret->array_data;
     886           2 :                 data += (index_offset * ret->count) * ret->memory_size;
     887           2 :                 b = COLnew(seqbase, TYPE_blob, (BUN)ret->count, TRANSIENT);
     888           2 :                 b->tnil = false;
     889           2 :                 b->tnonil = true;
     890           2 :                 b->tkey = false;
     891           2 :                 b->tsorted = false;
     892           2 :                 b->trevsorted = false;
     893           8 :                 for (iu = 0; iu < ret->count; iu++) {
     894             : 
     895           6 :                         char* memcpy_data;
     896           6 :                         size_t blob_len = 0;
     897             : 
     898           6 :                         if (ret->result_type == NPY_OBJECT) {
     899           0 :                                 PyObject *object = *((PyObject **)&data[0]);
     900           0 :                                 if (PyByteArray_Check(object)) {
     901           0 :                                         memcpy_data = PyByteArray_AsString(object);
     902           0 :                                         blob_len = pyobject_get_size(object);
     903             :                                 } else {
     904           0 :                                         pickle = PyObject_CallMethod(pickle_module, "dumps", "O", object);
     905           0 :                                         if (pickle == NULL) {
     906           0 :                                                 msg = createException(MAL, "pyapi3.eval",
     907             :                                                                                           SQLSTATE(PY000) "Can't pickle object to blob");
     908           0 :                                                 Py_XDECREF(pickle_module);
     909           0 :                                                 Python_ReleaseGIL(gstate);
     910           0 :                                                 goto wrapup;
     911             :                                         }
     912           0 :                                         memcpy_data = PyBytes_AsString(pickle);
     913           0 :                                         blob_len = pyobject_get_size(pickle);
     914           0 :                                         Py_XDECREF(pickle);
     915             :                                 }
     916           0 :                                 if (memcpy_data == NULL) {
     917           0 :                                         msg = createException(MAL, "pyapi3.eval",
     918             :                                                                                   SQLSTATE(PY000) "Can't get blob pickled object as char*");
     919           0 :                                         Py_XDECREF(pickle_module);
     920           0 :                                         Python_ReleaseGIL(gstate);
     921           0 :                                         goto wrapup;
     922             :                                 }
     923             :                         } else {
     924             :                                 memcpy_data = data;
     925             :                         }
     926             : 
     927           6 :                         if (mask && mask[iu]) {
     928           0 :                                 ele_blob = (blob *)GDKmalloc(offsetof(blob, data));
     929           0 :                                 ele_blob->nitems = ~(size_t)0;
     930             :                         } else {
     931           6 :                                 if (blob_fixed_size > 0) {
     932           6 :                                         blob_len = blob_fixed_size;
     933             :                                 }
     934           6 :                                 ele_blob = GDKmalloc(blobsize(blob_len));
     935           6 :                                 ele_blob->nitems = blob_len;
     936           6 :                                 memcpy(ele_blob->data, memcpy_data, blob_len);
     937             :                         }
     938           6 :                         if (BUNappend(b, ele_blob, FALSE) != GDK_SUCCEED) {
     939           0 :                                 if (ret->result_type == NPY_OBJECT) {
     940           0 :                                         Py_XDECREF(pickle_module);
     941           0 :                                         Python_ReleaseGIL(gstate);
     942             :                                 }
     943           0 :                                 BBPunfix(b->batCacheid);
     944           0 :                                 msg = createException(MAL, "pyapi3.eval", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     945           0 :                                 goto wrapup;
     946             :                         }
     947           6 :                         GDKfree(ele_blob);
     948           6 :                         data += ret->memory_size;
     949             : 
     950             :                 }
     951             : 
     952             :                 // We are done, we can release the GIL
     953           2 :                 if (ret->result_type == NPY_OBJECT) {
     954           0 :                         Py_XDECREF(pickle_module);
     955           0 :                         Python_ReleaseGIL(gstate);
     956             :                 }
     957             : 
     958           2 :                 BATsetcount(b, (BUN)ret->count);
     959           2 :                 BATsettrivprop(b);
     960             :         } else {
     961         170 :                 switch (bat_type) {
     962           0 :                         case TYPE_void:
     963           0 :                                 NP_CREATE_EMPTY_BAT(b, oid);
     964           0 :                                 break;
     965          11 :                         case TYPE_bit:
     966          11 :                                 NP_CREATE_BAT(b, bit);
     967             :                                 break;
     968           1 :                         case TYPE_bte:
     969         102 :                                 NP_CREATE_BAT(b, bte);
     970             :                                 break;
     971           1 :                         case TYPE_sht:
     972      100002 :                                 NP_CREATE_BAT(b, sht);
     973             :                                 break;
     974         114 :                         case TYPE_int:
     975    20475600 :                                 NP_CREATE_BAT(b, int);
     976             :                                 break;
     977           0 :                         case TYPE_oid:
     978           0 :                                 NP_CREATE_BAT(b, oid);
     979             :                                 break;
     980           1 :                         case TYPE_lng:
     981           1 :                                 NP_CREATE_BAT(b, lng);
     982             :                                 break;
     983           0 :                         case TYPE_flt:
     984           0 :                                 NP_CREATE_BAT(b, flt);
     985             :                                 break;
     986          23 :                         case TYPE_dbl:
     987      400146 :                                 NP_CREATE_BAT(b, dbl);
     988             :                                 break;
     989             : #ifdef HAVE_HGE
     990           0 :                         case TYPE_hge:
     991           0 :                                 NP_CREATE_BAT(b, hge);
     992             :                                 break;
     993             : #endif
     994          19 :                         case TYPE_str: {
     995          19 :                                 bool *mask = NULL;
     996          19 :                                 char *data = NULL;
     997          19 :                                 char *utf8_string = NULL;
     998          19 :                                 if (ret->mask_data != NULL) {
     999             :                                         mask = (bool *)ret->mask_data;
    1000             :                                 }
    1001          19 :                                 if (ret->array_data == NULL) {
    1002           0 :                                         msg = createException(
    1003             :                                                 MAL, "pyapi3.eval",
    1004             :                                                 SQLSTATE(PY000) "No return value stored in the structure.  n");
    1005           0 :                                         goto wrapup;
    1006             :                                 }
    1007          19 :                                 data = (char *)ret->array_data;
    1008             : 
    1009          19 :                                 if (ret->result_type != NPY_OBJECT) {
    1010          28 :                                         utf8_string =
    1011          14 :                                                 GDKzalloc(utf8string_minlength + ret->memory_size + 1);
    1012          14 :                                         utf8_string[utf8string_minlength + ret->memory_size] = '\0';
    1013             :                                 }
    1014             : 
    1015          19 :                                 b = COLnew(seqbase, TYPE_str, (BUN)ret->count, TRANSIENT);
    1016          19 :                                 b->tnil = false;
    1017          19 :                                 b->tnonil = true;
    1018          19 :                                 b->tkey = false;
    1019          19 :                                 b->tsorted = false;
    1020          19 :                                 b->trevsorted = false;
    1021     1100100 :                                 NP_INSERT_STRING_BAT(b);
    1022          19 :                                 if (utf8_string)
    1023          19 :                                         GDKfree(utf8_string);
    1024          19 :                                 BATsetcount(b, (BUN)ret->count);
    1025          19 :                                 BATsettrivprop(b);
    1026          19 :                                 break;
    1027             :                         }
    1028           0 :                         default:
    1029           0 :                                 msg = createException(MAL, "pyapi3.eval",
    1030             :                                                                           SQLSTATE(PY000) "Unrecognized BAT type %s.\n",
    1031             :                                                                           BatType_Format(bat_type));
    1032           0 :                                 goto wrapup;
    1033             :                 }
    1034             :         }
    1035         172 :         if (ConvertableSQLType(type)) {
    1036           6 :                 BAT *result;
    1037           6 :                 msg = ConvertToSQLType(NULL, b, type, &result, &bat_type);
    1038           6 :                 if (msg != MAL_SUCCEED) {
    1039           0 :                         goto wrapup;
    1040             :                 }
    1041           6 :                 b = result;
    1042             :         }
    1043             : 
    1044             :         return b;
    1045             : 
    1046           0 :   wrapup:
    1047           0 :         *return_message = msg;
    1048           0 :         return NULL;
    1049             : }
    1050             : 
    1051         464 : bit ConvertableSQLType(sql_subtype *sql_subtype)
    1052             : {
    1053         464 :         switch (GetSQLType(sql_subtype)) {
    1054             :                 case EC_DATE:
    1055             :                 case EC_TIME:
    1056             :                 case EC_TIMESTAMP:
    1057             :                 case EC_DEC:
    1058             :                         return 1;
    1059             :         }
    1060         456 :         return 0;
    1061             : }
    1062             : 
    1063         636 : int GetSQLType(sql_subtype *sql_subtype)
    1064             : {
    1065         636 :         if (!sql_subtype)
    1066             :                 return -1;
    1067         429 :         if (!sql_subtype->type)
    1068             :                 return -1;
    1069         429 :         return (int) sql_subtype->type->eclass;
    1070             : }
    1071             : 
    1072           7 : str ConvertFromSQLType(BAT *b, sql_subtype *sql_subtype, BAT **ret_bat,
    1073             :                                            int *ret_type)
    1074             : {
    1075           7 :         str res = MAL_SUCCEED;
    1076           7 :         int conv_type;
    1077             : 
    1078           7 :         assert(sql_subtype);
    1079           7 :         assert(sql_subtype->type);
    1080             : 
    1081           7 :         switch (sql_subtype->type->eclass) {
    1082             :                 case EC_DATE:
    1083             :                 case EC_TIME:
    1084             :                 case EC_TIMESTAMP:
    1085             :                         conv_type = TYPE_str;
    1086             :                         break;
    1087             :                 case EC_DEC:
    1088           2 :                         conv_type = TYPE_dbl;
    1089           2 :                         break;
    1090             :                 default:
    1091             :                         conv_type = TYPE_str;
    1092             :         }
    1093             : 
    1094           7 :         if (conv_type == TYPE_str) {
    1095           5 :                 BATiter li = bat_iterator(b);
    1096           5 :                 BUN p = 0, q = 0;
    1097           5 :                 char *result = NULL;
    1098           5 :                 size_t length = 0;
    1099           5 :                 ssize_t (*strConversion)(str *, size_t *, const void *, bool) =
    1100           5 :                         BATatoms[b->ttype].atomToStr;
    1101           5 :                 *ret_bat = COLnew(0, TYPE_str, 0, TRANSIENT);
    1102           5 :                 *ret_type = conv_type;
    1103           5 :                 if (!(*ret_bat)) {
    1104           0 :                         return createException(MAL, "pyapi3.eval",
    1105             :                                                                    SQLSTATE(HY013) MAL_MALLOC_FAIL " string conversion BAT.");
    1106             :                 }
    1107          11 :                 BATloop(b, p, q)
    1108             :                 {
    1109           6 :                         void *element = (void *)BUNtail(li, p);
    1110           6 :                         if (strConversion(&result, &length, element, false) < 0) {
    1111           0 :                                 BBPunfix((*ret_bat)->batCacheid);
    1112           0 :                                 return createException(MAL, "pyapi3.eval",
    1113             :                                                                            SQLSTATE(PY000) "Failed to convert element to string.");
    1114             :                         }
    1115           6 :                         if (BUNappend(*ret_bat, result, false) != GDK_SUCCEED) {
    1116           0 :                                 BBPunfix((*ret_bat)->batCacheid);
    1117           0 :                                 throw(MAL, "pyapi3.eval", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1118             :                         }
    1119             :                 }
    1120           5 :                 if (result) {
    1121           5 :                         GDKfree(result);
    1122             :                 }
    1123           5 :                 return res;
    1124           2 :         } else if (conv_type == TYPE_dbl) {
    1125           2 :                 int bat_type = ATOMstorage(b->ttype);
    1126           2 :                 int hpos = sql_subtype->scale;
    1127           2 :                 bat result = 0;
    1128             :                 // decimal values can be stored in various numeric fields, so check the
    1129             :                 // numeric field and convert the one it's actually stored in
    1130           2 :                 switch (bat_type) {
    1131           0 :                         case TYPE_bte:
    1132           0 :                                 res = (*batbte_dec2_dbl_ptr)(&result, &hpos, &b->batCacheid);
    1133           0 :                                 break;
    1134           0 :                         case TYPE_sht:
    1135           0 :                                 res = (*batsht_dec2_dbl_ptr)(&result, &hpos, &b->batCacheid);
    1136           0 :                                 break;
    1137           0 :                         case TYPE_int:
    1138           0 :                                 res = (*batint_dec2_dbl_ptr)(&result, &hpos, &b->batCacheid);
    1139           0 :                                 break;
    1140           2 :                         case TYPE_lng:
    1141           2 :                                 res = (*batlng_dec2_dbl_ptr)(&result, &hpos, &b->batCacheid);
    1142           2 :                                 break;
    1143             : #ifdef HAVE_HGE
    1144           0 :                         case TYPE_hge:
    1145           0 :                                 res = (*bathge_dec2_dbl_ptr)(&result, &hpos, &b->batCacheid);
    1146           0 :                                 break;
    1147             : #endif
    1148           0 :                         default:
    1149           0 :                                 return createException(MAL, "pyapi3.eval",
    1150             :                                                                            "Unsupported decimal storage type.");
    1151             :                 }
    1152           2 :                 if (res == MAL_SUCCEED) {
    1153           2 :                         *ret_bat = BATdescriptor(result);
    1154           2 :                         *ret_type = TYPE_dbl;
    1155             :                 } else {
    1156           0 :                         *ret_bat = NULL;
    1157             :                 }
    1158           2 :                 return res;
    1159             :         }
    1160             :         return createException(MAL, "pyapi3.eval", "Unrecognized conv type.");
    1161             : }
    1162             : 
    1163           6 : str ConvertToSQLType(Client cntxt, BAT *b, sql_subtype *sql_subtype,
    1164             :                                          BAT **ret_bat, int *ret_type)
    1165             : {
    1166           6 :         str res = MAL_SUCCEED;
    1167           6 :         bat result_bat = 0;
    1168           6 :         int digits = sql_subtype->digits;
    1169           6 :         int scale = sql_subtype->scale;
    1170           6 :         (void)cntxt;
    1171             : 
    1172           6 :         assert(sql_subtype);
    1173           6 :         assert(sql_subtype->type);
    1174             : 
    1175           6 :         switch (sql_subtype->type->eclass) {
    1176           1 :                 case EC_TIMESTAMP:
    1177           1 :                         res = (*batstr_2time_timestamp_ptr)(&result_bat, &b->batCacheid,
    1178             :                                                                                                 &digits);
    1179           1 :                         break;
    1180           1 :                 case EC_TIME:
    1181           1 :                         res = (*batstr_2time_daytime_ptr)(&result_bat, &b->batCacheid,
    1182             :                                                                                           &digits);
    1183           1 :                         break;
    1184           1 :                 case EC_DATE:
    1185           1 :                         res = (*batstr_2_date_ptr)(&result_bat, &b->batCacheid);
    1186           1 :                         break;
    1187           3 :                 case EC_DEC:
    1188           3 :                         res = (*batdbl_num2dec_lng_ptr)(&result_bat, &b->batCacheid,
    1189             :                                                                                         &digits, &scale);
    1190           3 :                         break;
    1191           0 :                 default:
    1192           0 :                         return createException(
    1193             :                                 MAL, "pyapi3.eval",
    1194             :                                 "Convert To SQL Type: Unrecognized SQL type %s (%d).",
    1195             :                                 sql_subtype->type->sqlname, (int) sql_subtype->type->eclass);
    1196             :         }
    1197           6 :         if (res == MAL_SUCCEED) {
    1198           6 :                 *ret_bat = BATdescriptor(result_bat);
    1199           6 :                 *ret_type = (*ret_bat)->ttype;
    1200             :         }
    1201             : 
    1202             :         return res;
    1203             : }
    1204             : 
    1205         224 : ssize_t PyType_Size(PyObject *obj)
    1206             : {
    1207         224 :         if (PyType_IsPyScalar(obj)) {
    1208             :                 return 1;
    1209             :         }
    1210          62 :         if (PyArray_Check(obj)) {
    1211          57 :                 return PyArray_Size(obj);
    1212             :         }
    1213           5 :         if (PyList_Check(obj)) {
    1214           5 :                 return Py_SIZE(obj);
    1215             :         }
    1216             :         return -1;
    1217             : }
    1218             : 
    1219         297 : bit IsStandardBATType(int type)
    1220             : {
    1221         297 :         switch (type) {
    1222             :                 case TYPE_bit:
    1223             :                 case TYPE_bte:
    1224             :                 case TYPE_sht:
    1225             :                 case TYPE_int:
    1226             :                 case TYPE_oid:
    1227             :                 case TYPE_lng:
    1228             :                 case TYPE_flt:
    1229             :                 case TYPE_dbl:
    1230             : #ifdef HAVE_HGE
    1231             :                 case TYPE_hge:
    1232             : #endif
    1233             :                 case TYPE_str:
    1234             :                         return 1;
    1235           5 :                 default:
    1236           5 :                         return 0;
    1237             :         }
    1238             : }
    1239             : 
    1240           7 : static void conversion_import_array(void) { _import_array(); }
    1241             : 
    1242           7 : str _conversion_init(void)
    1243             : {
    1244           7 :         str msg = MAL_SUCCEED;
    1245           7 :         conversion_import_array();
    1246             : 
    1247             : #ifndef HAVE_EMBEDDED
    1248           7 :         LOAD_SQL_FUNCTION_PTR(batbte_dec2_dbl);
    1249           7 :         LOAD_SQL_FUNCTION_PTR(batsht_dec2_dbl);
    1250           7 :         LOAD_SQL_FUNCTION_PTR(batint_dec2_dbl);
    1251           7 :         LOAD_SQL_FUNCTION_PTR(batlng_dec2_dbl);
    1252             : #ifdef HAVE_HGE
    1253           7 :         LOAD_SQL_FUNCTION_PTR(bathge_dec2_dbl);
    1254             : #endif
    1255           7 :         LOAD_SQL_FUNCTION_PTR(batstr_2time_timestamp);
    1256           7 :         LOAD_SQL_FUNCTION_PTR(batstr_2time_daytime);
    1257           7 :         LOAD_SQL_FUNCTION_PTR(batstr_2_date);
    1258           7 :         LOAD_SQL_FUNCTION_PTR(batdbl_num2dec_lng);
    1259             : #endif
    1260           7 :         return msg;
    1261             : }

Generated by: LCOV version 1.14