LCOV - code coverage report
Current view: top level - sql/backends/monet5/UDF/pyapi3 - conversion3.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 360 592 60.8 %
Date: 2021-10-13 02:24:04 Functions: 16 16 100.0 %

          Line data    Source code
       1             : /*
       2             :  * This Source Code Form is subject to the terms of the Mozilla Public
       3             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       4             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       5             :  *
       6             :  * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V.
       7             :  */
       8             : 
       9             : #include "monetdb_config.h"
      10             : #include "conversion.h"
      11             : #include "convert_loops.h"
      12             : #include "pytypes.h"
      13             : #include "type_conversion.h"
      14             : #include "unicode.h"
      15             : #include "blob.h"
      16             : #include "gdk_interprocess.h"
      17             : 
      18             : //! Wrapper to get eclass of SQL type
      19             : int GetSQLType(sql_subtype *sql_subtype);
      20             : 
      21             : static bool IsBlobType(int type)
      22             : {
      23         790 :         return type == TYPE_blob;
      24             : }
      25             : 
      26             : static bool IsVoidType(int type)
      27             : {
      28             :         return type == TYPE_void;
      29             : }
      30             : 
      31          92 : PyObject *PyArrayObject_FromScalar(PyInput *inp, char **return_message)
      32             : {
      33             :         PyObject *vararray = NULL;
      34             :         char *msg = NULL;
      35          92 :         assert(inp->scalar); // input has to be a scalar
      36             : 
      37          92 :         switch (inp->bat_type) {
      38           0 :                 case TYPE_void:
      39             : #if SIZEOF_OID == SIZEOF_INT
      40             :                         vararray = PyArray_Arange(0, 1, 1, NPY_UINT);
      41             : #else
      42           0 :                         vararray = PyArray_Arange(0, 1, 1, NPY_ULONGLONG);
      43             : #endif
      44           0 :                         break;
      45           0 :                 case TYPE_oid:
      46           0 :                         vararray = PyInt_FromLong((long)(*(oid *)inp->dataptr));
      47           0 :                         break;
      48          10 :                 case TYPE_bit:
      49          10 :                         vararray = PyInt_FromLong((long)(*(bit *)inp->dataptr));
      50          10 :                         break;
      51           0 :                 case TYPE_bte:
      52           0 :                         vararray = PyInt_FromLong((long)(*(bte *)inp->dataptr));
      53           0 :                         break;
      54           0 :                 case TYPE_sht:
      55           0 :                         vararray = PyInt_FromLong((long)(*(sht *)inp->dataptr));
      56           0 :                         break;
      57          63 :                 case TYPE_int:
      58          63 :                         vararray = PyInt_FromLong((long)(*(int *)inp->dataptr));
      59          63 :                         break;
      60           0 :                 case TYPE_lng:
      61           0 :                         vararray = PyLong_FromLongLong((*(lng *)inp->dataptr));
      62           0 :                         break;
      63           0 :                 case TYPE_flt:
      64           0 :                         vararray = PyFloat_FromDouble((double)(*(flt *)inp->dataptr));
      65           0 :                         break;
      66           1 :                 case TYPE_dbl:
      67           1 :                         vararray = PyFloat_FromDouble((double)(*(dbl *)inp->dataptr));
      68           1 :                         break;
      69             : #ifdef HAVE_HGE
      70           0 :                 case TYPE_hge:
      71           0 :                         vararray = PyLong_FromHge(*((hge *)inp->dataptr));
      72           0 :                         break;
      73             : #endif
      74          18 :                 case TYPE_str:
      75          18 :                         vararray = PyUnicode_FromString(*((char **)inp->dataptr));
      76          18 :                         break;
      77           0 :                 default:
      78           0 :                         msg = createException(MAL, "pyapi3.eval",
      79             :                                                                   SQLSTATE(PY000) "Unsupported scalar type %i.", inp->bat_type);
      80           0 :                         goto wrapup;
      81             :         }
      82          92 :         if (vararray == NULL) {
      83           0 :                 msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Something went wrong "
      84             :                                                                                                  "converting the MonetDB "
      85             :                                                                                                  "scalar to a Python scalar.");
      86           0 :                 goto wrapup;
      87             :         }
      88          92 : wrapup:
      89          92 :         *return_message = msg;
      90          92 :         return vararray;
      91             : }
      92             : 
      93         311 : PyObject *PyMaskedArray_FromBAT(PyInput *inp, size_t t_start, size_t t_end,
      94             :                                                                 char **return_message, bool copy)
      95             : {
      96             :         BAT *b;
      97             :         char *msg;
      98             :         PyObject *vararray;
      99             : 
     100         311 :         vararray = PyArrayObject_FromBAT(inp, t_start, t_end, return_message, copy);
     101         311 :         if (vararray == NULL) {
     102             :                 return NULL;
     103             :         }
     104         311 :         b = inp->bat;
     105             :         // To deal with null values, we use the numpy masked array structure
     106             :         // The masked array structure is an object with two arrays of equal size, a
     107             :         // data array and a mask array
     108             :         // The mask array is a boolean array that has the value 'True' when the
     109             :         // element is NULL, and 'False' otherwise
     110             :         // if we know for sure that the BAT has no NULL values, we can skip the construction
     111             :         // of this masked array. Otherwise, we create it.
     112         311 :         if (b->tnil || !b->tnonil) {
     113             :                 PyObject *mask;
     114         277 :                 PyObject *mafunc = PyObject_GetAttrString(
     115             :                         PyImport_Import(PyString_FromString("numpy.ma")), "masked_array");
     116         277 :                 PyObject *nullmask = PyNullMask_FromBAT(b, t_start, t_end);
     117             : 
     118         277 :                 if (!nullmask) {
     119           0 :                         msg = createException(MAL, "pyapi3.eval", "Failed to create mask for some reason");
     120           0 :                         goto wrapup;
     121         277 :                 } else if (nullmask == Py_None) {
     122             :                         Py_DECREF(nullmask);
     123             :                 } else {
     124          24 :                         PyObject *maargs = PyTuple_New(2);
     125          24 :                         PyTuple_SetItem(maargs, 0, vararray);
     126          24 :                         PyTuple_SetItem(maargs, 1, nullmask);
     127             : 
     128             :                         // Now we will actually construct the mask by calling the masked
     129             :                         // array constructor
     130          24 :                         mask = PyObject_CallObject(mafunc, maargs);
     131          24 :                         if (!mask) {
     132           0 :                                 msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Failed to create mask");
     133           0 :                                 goto wrapup;
     134             :                         }
     135             :                         Py_DECREF(maargs);
     136             : 
     137             :                         vararray = mask;
     138             :                 }
     139             :                 Py_DECREF(mafunc);
     140             :         }
     141             :         return vararray;
     142           0 : wrapup:
     143           0 :         *return_message = msg;
     144           0 :         return NULL;
     145             : }
     146             : 
     147         311 : PyObject *PyArrayObject_FromBAT(PyInput *inp, size_t t_start, size_t t_end,
     148             :                                                                 char **return_message, bool copy)
     149             : {
     150             :         // This variable will hold the converted Python object
     151             :         PyObject *vararray = NULL;
     152             :         char *msg;
     153             :         size_t j = 0;
     154             :         BUN p = 0, q = 0;
     155             :         BATiter li;
     156         311 :         BAT *b = inp->bat;
     157         311 :         npy_intp elements[1] = {t_end - t_start};
     158             : 
     159         311 :         assert(!inp->scalar); // input has to be a BAT
     160             : 
     161         311 :         if (!b) {
     162             :                 // No BAT was found, we can't do anything in this case
     163           0 :                 msg = createException(MAL, "pyapi3.eval", SQLSTATE(HY013) MAL_MALLOC_FAIL " bat missing");
     164           0 :                 goto wrapup;
     165             :         }
     166             : 
     167         612 :         if (!IsVoidType(inp->bat_type) && !IsBlobType(inp->bat_type) &&
     168         597 :                 (!IsStandardBATType(inp->bat_type) ||
     169         296 :                  ConvertableSQLType(inp->sql_subtype))) { // if the sql type is set, we
     170             :                                                                                                   // have to do some conversion
     171           7 :                 if (inp->scalar) {
     172             :                         // FIXME: scalar SQL types
     173           0 :                         msg = createException(
     174             :                                 MAL, "pyapi3.eval",
     175             :                                 SQLSTATE(0A000) "Scalar SQL types haven't been implemented yet... sorry");
     176           0 :                         goto wrapup;
     177             :                 } else {
     178           7 :                         BAT *ret_bat = NULL;
     179           7 :                         msg = ConvertFromSQLType(inp->bat, inp->sql_subtype, &ret_bat,
     180             :                                                                          &inp->bat_type);
     181           7 :                         if (msg != MAL_SUCCEED) {
     182           0 :                                 freeException(msg);
     183           0 :                                 msg = createException(MAL, "pyapi3.eval",
     184             :                                                                           SQLSTATE(PY000) "Failed to convert BAT.");
     185           0 :                                 goto wrapup;
     186             :                         }
     187           7 :                         b = ret_bat;
     188             :                 }
     189             :         }
     190             : 
     191         311 :         if (IsBlobType(inp->bat_type)) {
     192             :                 PyObject **data;
     193           2 :                 li = bat_iterator(b);
     194           2 :                 vararray = PyArray_EMPTY(1, elements, NPY_OBJECT, 0);
     195             :                 data = PyArray_DATA((PyArrayObject *)vararray);
     196           8 :                 BATloop(b, p, q)
     197             :                 {
     198           6 :                         blob *t = (blob *)BUNtvar(li, p);
     199           6 :                         if (t->nitems == ~(size_t)0) {
     200           0 :                                 data[p] = Py_None;
     201             :                                 Py_INCREF(Py_None);
     202             :                         } else {
     203           6 :                                 data[p] = PyByteArray_FromStringAndSize(t->data, t->nitems);
     204             :                         }
     205             :                 }
     206           2 :                 bat_iterator_end(&li);
     207             :         } else {
     208         309 :                 switch (inp->bat_type) {
     209           8 :                         case TYPE_void:
     210             : #if SIZEOF_OID == SIZEOF_INT
     211             :                                 BAT_TO_NP_CREATE_ALWAYS(b, NPY_UINT);
     212             : #else
     213           8 :                                 BAT_TO_NP_CREATE_ALWAYS(b, NPY_ULONGLONG);
     214             : #endif
     215           8 :                                 break;
     216          12 :                         case TYPE_oid:
     217             : #if SIZEOF_OID == SIZEOF_INT
     218             :                                 BAT_TO_NP(b, oid, NPY_UINT32);
     219             : #else
     220          12 :                                 BAT_TO_NP(b, oid, NPY_UINT64);
     221             : #endif
     222          12 :                                 break;
     223           1 :                         case TYPE_bit:
     224           1 :                                 BAT_TO_NP(b, bit, NPY_INT8);
     225           1 :                                 break;
     226           1 :                         case TYPE_bte:
     227           1 :                                 BAT_TO_NP(b, bte, NPY_INT8);
     228           1 :                                 break;
     229           0 :                         case TYPE_sht:
     230           0 :                                 BAT_TO_NP(b, sht, NPY_INT16);
     231           0 :                                 break;
     232         266 :                         case TYPE_int:
     233         266 :                                 BAT_TO_NP(b, int, NPY_INT32);
     234         266 :                                 break;
     235           0 :                         case TYPE_lng:
     236           0 :                                 BAT_TO_NP(b, lng, NPY_INT64);
     237           0 :                                 break;
     238           0 :                         case TYPE_flt:
     239           0 :                                 BAT_TO_NP(b, flt, NPY_FLOAT32);
     240           0 :                                 break;
     241          10 :                         case TYPE_dbl:
     242          10 :                                 BAT_TO_NP(b, dbl, NPY_FLOAT64);
     243          10 :                                 break;
     244          11 :                         case TYPE_str: {
     245             :                                 bool unicode = false;
     246          11 :                                 li = bat_iterator(b);
     247             :                                 // create a NPY_OBJECT array object
     248          11 :                                 vararray = PyArray_New(&PyArray_Type, 1, elements, NPY_OBJECT,
     249             :                                                                            NULL, NULL, 0, 0, NULL);
     250             : 
     251     1000040 :                                 BATloop(b, p, q)
     252             :                                 {
     253     1000030 :                                         char *t = (char *)BUNtvar(li, p);
     254     6000200 :                                         for (; *t != 0; t++) {
     255     5000171 :                                                 if (*t & 0x80) {
     256             :                                                         unicode = true;
     257             :                                                         break;
     258             :                                                 }
     259             :                                         }
     260     1000030 :                                         if (unicode) {
     261             :                                                 break;
     262             :                                         }
     263             :                                 }
     264             : 
     265             :                                 {
     266             :                                         PyObject **data =
     267             :                                                 ((PyObject **)PyArray_DATA((PyArrayObject *)vararray));
     268             :                                         PyObject *obj;
     269             :                                         j = 0;
     270          11 :                                         if (unicode) {
     271           1 :                                                 if (GDK_ELIMDOUBLES(b->tvheap)) {
     272             :                                                         PyObject **pyptrs =
     273           1 :                                                                 GDKzalloc(b->tvheap->free * sizeof(PyObject *));
     274           1 :                                                         if (!pyptrs) {
     275           0 :                                                                 bat_iterator_end(&li);
     276           0 :                                                                 msg = createException(MAL, "pyapi3.eval",
     277             :                                                                                                           SQLSTATE(HY013) MAL_MALLOC_FAIL
     278             :                                                                                                           " PyObject strings.");
     279           0 :                                                                 goto wrapup;
     280             :                                                         }
     281           3 :                                                         BATloop(b, p, q)
     282             :                                                         {
     283           2 :                                                                 const char *t = (const char *)BUNtvar(li, p);
     284           2 :                                                                 ptrdiff_t offset = t - b->tvheap->base;
     285           2 :                                                                 if (!pyptrs[offset]) {
     286           2 :                                                                         if (strNil(t)) {
     287             :                                                                                 // str_nil isn't a valid UTF-8 character
     288             :                                                                                 // (it's 0x80), so we can't decode it as
     289             :                                                                                 // UTF-8 (it will throw an error)
     290           1 :                                                                                 pyptrs[offset] =
     291           1 :                                                                                         PyUnicode_FromString("-");
     292             :                                                                         } else {
     293             :                                                                                 // otherwise we can just decode the
     294             :                                                                                 // string as UTF-8
     295           1 :                                                                                 pyptrs[offset] =
     296           1 :                                                                                         PyUnicode_FromString(t);
     297             :                                                                         }
     298           2 :                                                                         if (!pyptrs[offset]) {
     299           0 :                                                                                 bat_iterator_end(&li);
     300           0 :                                                                                 msg = createException(
     301             :                                                                                         MAL, "pyapi3.eval",
     302             :                                                                                         SQLSTATE(PY000) "Failed to create string.");
     303           0 :                                                                                 goto wrapup;
     304             :                                                                         }
     305             :                                                                 } else {
     306             :                                                                         Py_INCREF(pyptrs[offset]);
     307             :                                                                 }
     308           2 :                                                                 data[j++] = pyptrs[offset];
     309             :                                                         }
     310           1 :                                                         GDKfree(pyptrs);
     311             :                                                 } else {
     312           0 :                                                         BATloop(b, p, q)
     313             :                                                         {
     314           0 :                                                                 char *t = (char *)BUNtvar(li, p);
     315           0 :                                                                 if (strNil(t)) {
     316             :                                                                         // str_nil isn't a valid UTF-8 character
     317             :                                                                         // (it's 0x80), so we can't decode it as
     318             :                                                                         // UTF-8 (it will throw an error)
     319           0 :                                                                         obj = PyUnicode_FromString("-");
     320             :                                                                 } else {
     321             :                                                                         // otherwise we can just decode the string
     322             :                                                                         // as UTF-8
     323           0 :                                                                         obj = PyUnicode_FromString(t);
     324             :                                                                 }
     325             : 
     326           0 :                                                                 if (obj == NULL) {
     327           0 :                                                                         bat_iterator_end(&li);
     328           0 :                                                                         msg = createException(
     329             :                                                                                 MAL, "pyapi3.eval",
     330             :                                                                                 SQLSTATE(PY000) "Failed to create string.");
     331           0 :                                                                         goto wrapup;
     332             :                                                                 }
     333           0 :                                                                 data[j++] = obj;
     334             :                                                         }
     335             :                                                 }
     336             :                                         } else {
     337             :                                                 /* special case where we exploit the
     338             :                                                  * duplicate-eliminated string heap */
     339          10 :                                                 if (GDK_ELIMDOUBLES(b->tvheap)) {
     340             :                                                         PyObject **pyptrs =
     341          10 :                                                                 GDKzalloc(b->tvheap->free * sizeof(PyObject *));
     342          10 :                                                         if (!pyptrs) {
     343           0 :                                                                 bat_iterator_end(&li);
     344           0 :                                                                 msg = createException(MAL, "pyapi3.eval",
     345             :                                                                                                           SQLSTATE(HY013) MAL_MALLOC_FAIL
     346             :                                                                                                           " PyObject strings.");
     347           0 :                                                                 goto wrapup;
     348             :                                                         }
     349     1000038 :                                                         BATloop(b, p, q)
     350             :                                                         {
     351     1000028 :                                                                 const char *t = (const char *)BUNtvar(li, p);
     352     1000028 :                                                                 ptrdiff_t offset = t - b->tvheap->base;
     353     1000028 :                                                                 if (!pyptrs[offset]) {
     354          21 :                                                                         pyptrs[offset] = PyString_FromString(t);
     355             :                                                                 } else {
     356             :                                                                         Py_INCREF(pyptrs[offset]);
     357             :                                                                 }
     358     1000028 :                                                                 data[j++] = pyptrs[offset];
     359             :                                                         }
     360          10 :                                                         GDKfree(pyptrs);
     361             :                                                 } else {
     362           0 :                                                         BATloop(b, p, q)
     363             :                                                         {
     364           0 :                                                                 char *t = (char *)BUNtvar(li, p);
     365           0 :                                                                 obj = PyString_FromString(t);
     366           0 :                                                                 if (obj == NULL) {
     367           0 :                                                                         bat_iterator_end(&li);
     368           0 :                                                                         msg = createException(
     369             :                                                                                 MAL, "pyapi3.eval",
     370             :                                                                                 SQLSTATE(PY000) "Failed to create string.");
     371           0 :                                                                         goto wrapup;
     372             :                                                                 }
     373           0 :                                                                 data[j++] = obj;
     374             :                                                         }
     375             :                                                 }
     376             :                                         }
     377             :                                 }
     378          11 :                                 bat_iterator_end(&li);
     379          11 :                         } break;
     380             : #ifdef HAVE_HGE
     381           0 :                         case TYPE_hge: {
     382             :                                 // create a NPY_FLOAT64 array to hold the huge type
     383           0 :                                 vararray = PyArray_New(&PyArray_Type, 1,
     384           0 :                                                                            (npy_intp[1]){t_end - t_start},
     385             :                                                                            NPY_FLOAT64, NULL, NULL, 0, 0, NULL);
     386             : 
     387             :                                 j = 0;
     388             :                                 npy_float64 *data = (npy_float64 *)PyArray_DATA((PyArrayObject *)vararray);
     389           0 :                                 BATiter bi = bat_iterator(b);
     390           0 :                                 const hge *vals = (const hge *) bi.base;
     391           0 :                                 BATloop(b, p, q) {
     392           0 :                                         data[j++] = (npy_float64)vals[p];
     393             :                                 }
     394           0 :                                 bat_iterator_end(&bi);
     395             :                                 break;
     396             :                         }
     397             : #endif
     398           0 :                         default:
     399           0 :                                 if (!inp->sql_subtype || !inp->sql_subtype->type) {
     400           0 :                                         msg = createException(MAL, "pyapi3.eval",
     401             :                                                                                   SQLSTATE(PY000) "unknown argument type");
     402             :                                 } else {
     403           0 :                                         msg = createException(MAL, "pyapi3.eval",
     404             :                                                                                   SQLSTATE(PY000) "Unsupported SQL Type: %s",
     405             :                                                                                   inp->sql_subtype->type->base.name);
     406             :                                 }
     407           0 :                                 goto wrapup;
     408             :                 }
     409             :         }
     410             : 
     411         311 :         if (vararray == NULL) {
     412           0 :                 msg = createException(MAL, "pyapi3.eval",
     413             :                                                           SQLSTATE(PY000) "Failed to convert BAT to Numpy array.");
     414           0 :                 goto wrapup;
     415             :         }
     416         311 :         if (b != inp->bat)
     417           7 :                 BBPunfix(b->batCacheid);
     418             :         return vararray;
     419           0 : wrapup:
     420           0 :         *return_message = msg;
     421           0 :         if (b != inp->bat)
     422           0 :                 BBPunfix(b->batCacheid);
     423             :         return NULL;
     424             : }
     425             : 
     426             : #define CreateNullMask(tpe)                                                    \
     427             :         {                                                                          \
     428             :                 tpe *bat_ptr = (tpe *)b->theap->base + b->tbaseoff;                    \
     429             :                 for (j = 0; j < count; j++) {                                          \
     430             :                         mask_data[j] = is_##tpe##_nil(bat_ptr[j]);                         \
     431             :                         found_nil |= mask_data[j];                                         \
     432             :                 }                                                                      \
     433             :         }
     434             : 
     435         277 : PyObject *PyNullMask_FromBAT(BAT *b, size_t t_start, size_t t_end)
     436             : {
     437             :         // We will now construct the Masked array, we start by setting everything to
     438             :         // False
     439         277 :         size_t count = t_end - t_start;
     440         277 :         npy_intp elements[1] = {count};
     441             :         PyArrayObject *nullmask =
     442         277 :                 (PyArrayObject *)PyArray_EMPTY(1, elements, NPY_BOOL, 0);
     443         277 :         const void *nil = ATOMnilptr(b->ttype);
     444             :         size_t j;
     445             :         bool found_nil = false;
     446         277 :         BATiter bi = bat_iterator(b);
     447             :         bool *mask_data = (bool *)PyArray_DATA(nullmask);
     448             : 
     449         549 :         switch (ATOMbasetype(getBatType(b->ttype))) {
     450           0 :                 case TYPE_bit:
     451           0 :                         CreateNullMask(bit);
     452             :                         break;
     453           2 :                 case TYPE_bte:
     454          12 :                         CreateNullMask(bte);
     455             :                         break;
     456           0 :                 case TYPE_sht:
     457           0 :                         CreateNullMask(sht);
     458             :                         break;
     459         258 :                 case TYPE_int:
     460      136423 :                         CreateNullMask(int);
     461             :                         break;
     462           4 :                 case TYPE_lng:
     463           9 :                         CreateNullMask(lng);
     464             :                         break;
     465           0 :                 case TYPE_flt:
     466           0 :                         CreateNullMask(flt);
     467             :                         break;
     468           6 :                 case TYPE_dbl:
     469        2115 :                         CreateNullMask(dbl);
     470             :                         break;
     471             : #ifdef HAVE_HGE
     472           0 :                 case TYPE_hge:
     473           0 :                         CreateNullMask(hge);
     474             :                         break;
     475             : #endif
     476           7 :                 default: {
     477           7 :                         int (*atomcmp)(const void *, const void *) = ATOMcompare(b->ttype);
     478          31 :                         for (j = 0; j < count; j++) {
     479          24 :                                 mask_data[j] = (*atomcmp)(BUNtail(bi, (BUN)(j)), nil) == 0;
     480          24 :                                 found_nil |= mask_data[j];
     481             :                         }
     482             :                 }
     483             :         }
     484         277 :         bat_iterator_end(&bi);
     485             : 
     486         277 :         if (!found_nil) {
     487             :                 Py_DECREF(nullmask);
     488         253 :                 Py_RETURN_NONE;
     489             :         }
     490             : 
     491             :         return (PyObject *)nullmask;
     492             : }
     493             : 
     494          35 : PyObject *PyDict_CheckForConversion(PyObject *pResult, int expected_columns,
     495             :                                                                         char **retcol_names, char **return_message)
     496             : {
     497             :         char *msg = MAL_SUCCEED;
     498          35 :         PyObject *result = PyList_New(expected_columns),
     499          35 :                          *keys = PyDict_Keys(pResult);
     500             :         int i;
     501             : 
     502         115 :         for (i = 0; i < expected_columns; i++) {
     503          83 :                 PyObject *object = PyDict_GetItemString(pResult, retcol_names[i]);
     504          83 :                 if (object == NULL) {
     505             :                         msg =
     506           2 :                                 createException(MAL, "pyapi3.eval",
     507             :                                                                 SQLSTATE(PY000) "Expected a return value with name \"%s\", but "
     508             :                                                                 "this key was not present in the dictionary.",
     509             :                                                                 retcol_names[i]);
     510           2 :                         goto wrapup;
     511             :                 }
     512             :                 Py_INCREF(object);
     513          81 :                 object = PyObject_CheckForConversion(object, 1, NULL, return_message);
     514          81 :                 if (object == NULL) {
     515           1 :                         msg = createException(
     516             :                                 MAL, "pyapi3.eval",
     517             :                                 SQLSTATE(PY000) "Error converting dict return value \"%s\": %s.",
     518             :                                 retcol_names[i], getExceptionMessage(*return_message));
     519           1 :                         GDKfree(*return_message);
     520           1 :                         goto wrapup;
     521             :                 }
     522          80 :                 if (PyList_CheckExact(object)) {
     523          80 :                         PyObject *item = PyList_GetItem(object, 0);
     524          80 :                         PyList_SetItem(result, i, item);
     525             :                         Py_INCREF(item);
     526             :                         Py_DECREF(object);
     527             :                 } else {
     528           0 :                         msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Why is this not a list?");
     529           0 :                         goto wrapup;
     530             :                 }
     531             :         }
     532             :         Py_DECREF(keys);
     533             :         Py_DECREF(pResult);
     534             :         // Py_INCREF(result);
     535             :         return result;
     536           3 : wrapup:
     537           3 :         *return_message = msg;
     538             :         Py_DECREF(result);
     539             :         Py_DECREF(keys);
     540             :         Py_DECREF(pResult);
     541             :         return NULL;
     542             : }
     543             : 
     544         177 : PyObject *PyObject_CheckForConversion(PyObject *pResult, int expected_columns,
     545             :                                                                           int *actual_columns,
     546             :                                                                           char **return_message)
     547             : {
     548             :         char *msg;
     549             :         int columns = 0;
     550         177 :         if (pResult) {
     551             :                 PyObject *pColO = NULL;
     552         177 :                 if (PyType_IsPandasDataFrame(pResult)) {
     553             :                         // the result object is a Pandas data frame
     554             :                         // we can convert the pandas data frame to a numpy array by simply
     555             :                         // accessing the "values" field (as pandas dataframes are numpy
     556             :                         // arrays internally)
     557           0 :                         pResult = PyObject_GetAttrString(pResult, "values");
     558           0 :                         if (pResult == NULL) {
     559           0 :                                 msg = createException(MAL, "pyapi3.eval",
     560             :                                                                           SQLSTATE(PY000) "Invalid Pandas data frame.");
     561           0 :                                 goto wrapup;
     562             :                         }
     563             :                         // we transpose the values field so it's aligned correctly for our
     564             :                         // purposes
     565           0 :                         pResult = PyObject_GetAttrString(pResult, "T");
     566           0 :                         if (pResult == NULL) {
     567           0 :                                 msg = createException(MAL, "pyapi3.eval",
     568             :                                                                           SQLSTATE(PY000) "Invalid Pandas data frame.");
     569           0 :                                 goto wrapup;
     570             :                         }
     571             :                 }
     572             : 
     573         177 :                 if (PyType_IsPyScalar(
     574             :                                 pResult)) { // check if the return object is a scalar
     575          59 :                         if (expected_columns == 1 || expected_columns <= 0) {
     576             :                                 // if we only expect a single return value, we can accept
     577             :                                 // scalars by converting it into an array holding an array
     578             :                                 // holding the element (i.e. [[pResult]])
     579          59 :                                 PyObject *list = PyList_New(1);
     580          59 :                                 PyList_SetItem(list, 0, pResult);
     581             :                                 pResult = list;
     582             : 
     583          59 :                                 list = PyList_New(1);
     584          59 :                                 PyList_SetItem(list, 0, pResult);
     585             :                                 pResult = list;
     586             : 
     587             :                                 columns = 1;
     588             :                         } else {
     589             :                                 // the result object is a scalar, yet we expect more than one
     590             :                                 // return value. We can only convert the result into a list with
     591             :                                 // a single element, so the output is necessarily wrong.
     592           0 :                                 msg = createException(
     593             :                                         MAL, "pyapi3.eval",
     594             :                                         SQLSTATE(PY000) "A single scalar was returned, yet we expect a list of %d "
     595             :                                         "columns. We can only convert a single scalar into a "
     596             :                                         "single column, thus the result is invalid.",
     597             :                                         expected_columns);
     598           0 :                                 goto wrapup;
     599             :                         }
     600             :                 } else {
     601             :                         // if it is not a scalar, we check if it is a single array
     602             :                         bool IsSingleArray = TRUE;
     603             :                         PyObject *data = pResult;
     604         118 :                         if (PyType_IsNumpyMaskedArray(data)) {
     605          30 :                                 data = PyObject_GetAttrString(pResult, "data");
     606          30 :                                 if (data == NULL) {
     607           0 :                                         msg = createException(MAL, "pyapi3.eval",
     608             :                                                                                   SQLSTATE(PY000) "Invalid masked array.");
     609           0 :                                         goto wrapup;
     610             :                                 }
     611             :                         }
     612         118 :                         if (PyType_IsNumpyArray(data)) {
     613          84 :                                 if (PyArray_NDIM((PyArrayObject *)data) != 1) {
     614             :                                         IsSingleArray = FALSE;
     615             :                                 } else {
     616             :                                         pColO = PyArray_GETITEM(
     617             :                                                 (PyArrayObject *)data,
     618             :                                                 PyArray_GETPTR1((PyArrayObject *)data, 0));
     619          84 :                                         IsSingleArray = PyType_IsPyScalar(pColO);
     620             :                                 }
     621          34 :                         } else if (PyList_Check(data)) {
     622          34 :                                 pColO = PyList_GetItem(data, 0);
     623          34 :                                 IsSingleArray = PyType_IsPyScalar(pColO);
     624           0 :                         } else if (!PyType_IsNumpyMaskedArray(data)) {
     625             :                                 // it is neither a python array, numpy array or numpy masked
     626             :                                 // array, thus the result is unsupported! Throw an exception!
     627           0 :                                 msg = createException(
     628             :                                         MAL, "pyapi3.eval",
     629             :                                         SQLSTATE(PY000) "Unsupported result object. Expected either a list, "
     630             :                                         "dictionary, a numpy array, a numpy masked array or a "
     631             :                                         "pandas data frame, but received an object of type \"%s\"",
     632             :                                         PyString_AsString(PyObject_Str(PyObject_Type(data))));
     633           0 :                                 goto wrapup;
     634             :                         }
     635             : 
     636         118 :                         if (IsSingleArray) {
     637         110 :                                 if (expected_columns == 1 || expected_columns <= 0) {
     638             :                                         // if we only expect a single return value, we can accept a
     639             :                                         // single array by converting it into an array holding an
     640             :                                         // array holding the element (i.e. [pResult])
     641         110 :                                         PyObject *list = PyList_New(1);
     642         110 :                                         PyList_SetItem(list, 0, pResult);
     643             :                                         pResult = list;
     644             : 
     645             :                                         columns = 1;
     646             :                                 } else {
     647             :                                         // the result object is a single array, yet we expect more
     648             :                                         // than one return value. We can only convert the result
     649             :                                         // into a list with a single array, so the output is
     650             :                                         // necessarily wrong.
     651           0 :                                         msg = createException(MAL, "pyapi3.eval",
     652             :                                                                                   SQLSTATE(PY000) "A single array was returned, yet we "
     653             :                                                                                   "expect a list of %d columns. The "
     654             :                                                                                   "result is invalid.",
     655             :                                                                                   expected_columns);
     656           0 :                                         goto wrapup;
     657             :                                 }
     658             :                         } else {
     659             :                                 // the return value is an array of arrays, all we need to do is
     660             :                                 // check if it is the correct size
     661             :                                 int results = 0;
     662           8 :                                 if (PyList_Check(data))
     663           8 :                                         results = (int)PyList_Size(data);
     664             :                                 else
     665           0 :                                         results = (int)PyArray_DIMS((PyArrayObject *)data)[0];
     666             :                                 columns = results;
     667           8 :                                 if (results != expected_columns && expected_columns > 0) {
     668             :                                         // wrong return size, we expect pci->retc arrays
     669           1 :                                         msg = createException(MAL, "pyapi3.eval",
     670             :                                                                                   SQLSTATE(PY000) "An array of size %d was returned, "
     671             :                                                                                   "yet we expect a list of %d columns. "
     672             :                                                                                   "The result is invalid.",
     673             :                                                                                   results, expected_columns);
     674           1 :                                         goto wrapup;
     675             :                                 }
     676             :                         }
     677             :                 }
     678             :         } else {
     679           0 :                 msg = createException(
     680             :                         MAL, "pyapi3.eval",
     681             :                         SQLSTATE(PY000) "Invalid result object. No result object could be generated.");
     682           0 :                 goto wrapup;
     683             :         }
     684             : 
     685         176 :         if (actual_columns != NULL)
     686           0 :                 *actual_columns = columns;
     687             :         return pResult;
     688           1 : wrapup:
     689           1 :         if (actual_columns != NULL)
     690           0 :                 *actual_columns = columns;
     691           1 :         *return_message = msg;
     692           1 :         return NULL;
     693             : }
     694             : 
     695         217 : str PyObject_GetReturnValues(PyObject *obj, PyReturn *ret)
     696             : {
     697             :         PyObject *pMask = NULL;
     698             :         str msg = MAL_SUCCEED;
     699             :         // If it isn't we need to convert pColO to the expected Numpy Array type
     700         217 :         ret->numpy_array = PyArray_FromAny(
     701             :                 obj, NULL, 1, 1, NPY_ARRAY_CARRAY | NPY_ARRAY_FORCECAST, NULL);
     702         217 :         if (ret->numpy_array == NULL) {
     703           0 :                 msg = createException(
     704             :                         MAL, "pyapi3.eval",
     705             :                         SQLSTATE(PY000) "Could not create a Numpy array from the return type.\n");
     706           0 :                 goto wrapup;
     707             :         }
     708             : 
     709         217 :         ret->result_type =
     710             :                 PyArray_DESCR((PyArrayObject *)ret->numpy_array)
     711         217 :                         ->type_num; // We read the result type from the resulting array
     712         217 :         ret->memory_size = PyArray_DESCR((PyArrayObject *)ret->numpy_array)->elsize;
     713         217 :         ret->count = PyArray_DIMS((PyArrayObject *)ret->numpy_array)[0];
     714         217 :         ret->array_data = PyArray_DATA((PyArrayObject *)ret->numpy_array);
     715         217 :         ret->mask_data = NULL;
     716         217 :         ret->numpy_mask = NULL;
     717             :         // If pColO is a Masked array, we convert the mask to a NPY_BOOL numpy array
     718         217 :         if (PyObject_HasAttrString(obj, "mask")) {
     719          30 :                 pMask = PyObject_GetAttrString(obj, "mask");
     720          30 :                 if (pMask != NULL) {
     721          30 :                         ret->numpy_mask =
     722          30 :                                 PyArray_FromAny(pMask, PyArray_DescrFromType(NPY_BOOL), 1, 1,
     723             :                                                                 NPY_ARRAY_CARRAY, NULL);
     724          30 :                         if (ret->numpy_mask == NULL ||
     725          30 :                                 PyArray_DIMS((PyArrayObject *)ret->numpy_mask)[0] !=
     726          30 :                                         (int)ret->count) {
     727           0 :                                 PyErr_Clear();
     728             :                                 pMask = NULL;
     729           0 :                                 ret->numpy_mask = NULL;
     730             :                         }
     731             :                 }
     732             :         }
     733         217 :         if (ret->numpy_mask != NULL)
     734          30 :                 ret->mask_data = PyArray_DATA((PyArrayObject *)ret->numpy_mask);
     735         187 : wrapup:
     736         217 :         return msg;
     737             : }
     738             : 
     739         128 : bool PyObject_PreprocessObject(PyObject *pResult, PyReturn *pyreturn_values,
     740             :                                                            int column_count, char **return_message)
     741             : {
     742             :         int i;
     743             :         char *msg;
     744         304 :         for (i = 0; i < column_count; i++) {
     745             :                 // Refers to the current Numpy mask (if it exists)
     746             :                 PyObject *pMask = NULL;
     747             :                 // Refers to the current Numpy array
     748             :                 PyObject *pColO = NULL;
     749             :                 // This is the PyReturn header information for the current return value,
     750             :                 // we will fill this now
     751         176 :                 PyReturn *ret = &pyreturn_values[i];
     752             : 
     753         176 :                 ret->multidimensional = FALSE;
     754             :                 // There are three possibilities (we have ensured this right after
     755             :                 // executing the Python call by calling PyObject_CheckForConversion)
     756             :                 // 1: The top level result object is a PyList or Numpy Array containing
     757             :                 // pci->retc Numpy Arrays
     758             :                 // 2: The top level result object is a (pci->retc x N) dimensional Numpy
     759             :                 // Array [Multidimensional]
     760             :                 // 3: The top level result object is a (pci->retc x N) dimensional Numpy
     761             :                 // Masked Array [Multidimensional]
     762         176 :                 if (PyList_Check(pResult)) {
     763             :                         // If it is a PyList, we simply get the i'th Numpy array from the
     764             :                         // PyList
     765         176 :                         pColO = PyList_GetItem(pResult, i);
     766             :                 } else {
     767             :                         // If it isn't, the result object is either a Nump Masked Array or a
     768             :                         // Numpy Array
     769             :                         PyObject *data = pResult;
     770           0 :                         if (PyType_IsNumpyMaskedArray(data)) {
     771           0 :                                 data = PyObject_GetAttrString(
     772             :                                         pResult, "data"); // If it is a Masked array, the data is
     773             :                                                                           // stored in the masked_array.data
     774             :                                                                           // attribute
     775           0 :                                 pMask = PyObject_GetAttrString(pResult, "mask");
     776             :                         }
     777             : 
     778             :                         // We can either have a multidimensional numpy array, or a single
     779             :                         // dimensional numpy array
     780           0 :                         if (PyArray_NDIM((PyArrayObject *)data) != 1) {
     781             :                                 // If it is a multidimensional numpy array, we have to convert
     782             :                                 // the i'th dimension to a NUMPY array object
     783           0 :                                 ret->multidimensional = TRUE;
     784           0 :                                 ret->result_type =
     785           0 :                                         PyArray_DESCR((PyArrayObject *)data)->type_num;
     786             :                         } else {
     787             :                                 // If it is a single dimensional Numpy array, we get the i'th
     788             :                                 // Numpy array from the Numpy Array
     789             :                                 pColO =
     790             :                                         PyArray_GETITEM((PyArrayObject *)data,
     791           0 :                                                                         PyArray_GETPTR1((PyArrayObject *)data, i));
     792             :                         }
     793             :                 }
     794             : 
     795             :                 // Now we have to do some preprocessing on the data
     796         176 :                 if (ret->multidimensional) {
     797             :                         // If it is a multidimensional Numpy array, we don't need to do any
     798             :                         // conversion, we can just do some pointers
     799           0 :                         ret->count = PyArray_DIMS((PyArrayObject *)pResult)[1];
     800           0 :                         ret->numpy_array = pResult;
     801           0 :                         ret->numpy_mask = pMask;
     802           0 :                         ret->array_data = PyArray_DATA((PyArrayObject *)ret->numpy_array);
     803           0 :                         if (ret->numpy_mask != NULL)
     804           0 :                                 ret->mask_data = PyArray_DATA((PyArrayObject *)ret->numpy_mask);
     805           0 :                         ret->memory_size =
     806           0 :                                 PyArray_DESCR((PyArrayObject *)ret->numpy_array)->elsize;
     807             :                 } else {
     808         176 :                         msg = PyObject_GetReturnValues(pColO, ret);
     809         176 :                         if (msg != MAL_SUCCEED) {
     810           0 :                                 goto wrapup;
     811             :                         }
     812             :                 }
     813             :         }
     814             :         return TRUE;
     815             : wrapup:
     816           0 :         *return_message = msg;
     817           0 :         return FALSE;
     818             : }
     819             : 
     820         176 : BAT *PyObject_ConvertToBAT(PyReturn *ret, sql_subtype *type, int bat_type,
     821             :                                                    int i, oid seqbase, char **return_message, bool copy)
     822             : {
     823             :         BAT *b = NULL;
     824             :         size_t index_offset = 0;
     825             :         char *msg;
     826             :         size_t iu;
     827             : 
     828         176 :         if (ret->multidimensional)
     829             :                 index_offset = i;
     830             : 
     831         176 :         switch (GetSQLType(type)) {
     832           3 :                 case EC_TIMESTAMP:
     833             :                 case EC_TIME:
     834             :                 case EC_DATE:
     835           3 :                         bat_type = TYPE_str;
     836           3 :                         break;
     837           3 :                 case EC_DEC:
     838           3 :                         bat_type = TYPE_dbl;
     839           3 :                         break;
     840             :                 default:
     841             :                         break;
     842             :         }
     843             : 
     844         176 :         if (IsBlobType(bat_type)) {
     845             :                 bool *mask = NULL;
     846             :                 char *data = NULL;
     847             :                 blob *ele_blob;
     848           2 :                 size_t blob_fixed_size = ret->memory_size;
     849             : 
     850             :                 PyObject *pickle_module = NULL, *pickle = NULL;
     851             :                 bool gstate = 0;
     852             : 
     853           2 :                 if (ret->result_type == NPY_OBJECT) {
     854             :                         // Python objects, we may need to pickle them, so we
     855             :                         // may execute Python code, we have to obtain the GIL
     856           0 :                         gstate = Python_ObtainGIL();
     857           0 :                         pickle_module = PyImport_ImportModule("pickle");
     858           0 :                         if (pickle_module == NULL) {
     859           0 :                                 msg = createException(MAL, "pyapi3.eval",
     860             :                                                                           SQLSTATE(PY000) "Can't load pickle module to pickle python object to blob");
     861           0 :                                 Python_ReleaseGIL(gstate);
     862           0 :                                 goto wrapup;
     863             :                         }
     864             :                         blob_fixed_size = 0; // Size depends on the objects
     865             :                 }
     866             : 
     867           2 :                 if (ret->mask_data != NULL) {
     868             :                         mask = (bool *)ret->mask_data;
     869             :                 }
     870           2 :                 if (ret->array_data == NULL) {
     871           0 :                         msg = createException(MAL, "pyapi3.eval",
     872             :                                                                   SQLSTATE(PY000) "No return value stored in the structure.");
     873           0 :                         if (ret->result_type == NPY_OBJECT) {
     874           0 :                                 Py_XDECREF(pickle_module);
     875           0 :                                 Python_ReleaseGIL(gstate);
     876             :                         }
     877           0 :                         goto wrapup;
     878             :                 }
     879             :                 data = (char *)ret->array_data;
     880           2 :                 data += (index_offset * ret->count) * ret->memory_size;
     881           2 :                 b = COLnew(seqbase, TYPE_blob, (BUN)ret->count, TRANSIENT);
     882           2 :                 b->tnil = false;
     883           2 :                 b->tnonil = true;
     884           2 :                 b->tkey = false;
     885           2 :                 b->tsorted = false;
     886           2 :                 b->trevsorted = false;
     887           8 :                 for (iu = 0; iu < ret->count; iu++) {
     888             : 
     889             :                         char* memcpy_data;
     890             :                         size_t blob_len = 0;
     891             : 
     892           6 :                         if (ret->result_type == NPY_OBJECT) {
     893           0 :                                 PyObject *object = *((PyObject **)&data[0]);
     894           0 :                                 if (PyByteArray_Check(object)) {
     895           0 :                                         memcpy_data = PyByteArray_AsString(object);
     896           0 :                                         blob_len = pyobject_get_size(object);
     897             :                                 } else {
     898           0 :                                         pickle = PyObject_CallMethod(pickle_module, "dumps", "O", object);
     899           0 :                                         if (pickle == NULL) {
     900           0 :                                                 msg = createException(MAL, "pyapi3.eval",
     901             :                                                                                           SQLSTATE(PY000) "Can't pickle object to blob");
     902           0 :                                                 Py_XDECREF(pickle_module);
     903           0 :                                                 Python_ReleaseGIL(gstate);
     904           0 :                                                 goto wrapup;
     905             :                                         }
     906           0 :                                         memcpy_data = PyBytes_AsString(pickle);
     907           0 :                                         blob_len = pyobject_get_size(pickle);
     908           0 :                                         Py_XDECREF(pickle);
     909             :                                 }
     910           0 :                                 if (memcpy_data == NULL) {
     911           0 :                                         msg = createException(MAL, "pyapi3.eval",
     912             :                                                                                   SQLSTATE(PY000) "Can't get blob pickled object as char*");
     913           0 :                                         Py_XDECREF(pickle_module);
     914           0 :                                         Python_ReleaseGIL(gstate);
     915           0 :                                         goto wrapup;
     916             :                                 }
     917             :                         } else {
     918             :                                 memcpy_data = data;
     919             :                         }
     920             : 
     921           6 :                         if (mask && mask[iu]) {
     922           0 :                                 ele_blob = (blob *)GDKmalloc(offsetof(blob, data));
     923           0 :                                 ele_blob->nitems = ~(size_t)0;
     924             :                         } else {
     925           6 :                                 if (blob_fixed_size > 0) {
     926             :                                         blob_len = blob_fixed_size;
     927             :                                 }
     928           6 :                                 ele_blob = GDKmalloc(blobsize(blob_len));
     929           6 :                                 ele_blob->nitems = blob_len;
     930           6 :                                 memcpy(ele_blob->data, memcpy_data, blob_len);
     931             :                         }
     932           6 :                         if (BUNappend(b, ele_blob, FALSE) != GDK_SUCCEED) {
     933           0 :                                 if (ret->result_type == NPY_OBJECT) {
     934           0 :                                         Py_XDECREF(pickle_module);
     935           0 :                                         Python_ReleaseGIL(gstate);
     936             :                                 }
     937           0 :                                 BBPunfix(b->batCacheid);
     938           0 :                                 msg = createException(MAL, "pyapi3.eval", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     939           0 :                                 goto wrapup;
     940             :                         }
     941           6 :                         GDKfree(ele_blob);
     942           6 :                         data += ret->memory_size;
     943             : 
     944             :                 }
     945             : 
     946             :                 // We are done, we can release the GIL
     947           2 :                 if (ret->result_type == NPY_OBJECT) {
     948           0 :                         Py_XDECREF(pickle_module);
     949           0 :                         Python_ReleaseGIL(gstate);
     950             :                 }
     951             : 
     952           2 :                 BATsetcount(b, (BUN)ret->count);
     953           2 :                 BATsettrivprop(b);
     954             :         } else {
     955         174 :                 switch (bat_type) {
     956           0 :                         case TYPE_void:
     957           0 :                                 NP_CREATE_EMPTY_BAT(b, oid);
     958           0 :                                 break;
     959          11 :                         case TYPE_bit:
     960          11 :                                 NP_CREATE_BAT(b, bit);
     961             :                                 break;
     962           1 :                         case TYPE_bte:
     963         102 :                                 NP_CREATE_BAT(b, bte);
     964             :                                 break;
     965           1 :                         case TYPE_sht:
     966      100002 :                                 NP_CREATE_BAT(b, sht);
     967             :                                 break;
     968         118 :                         case TYPE_int:
     969    20470637 :                                 NP_CREATE_BAT(b, int);
     970             :                                 break;
     971           0 :                         case TYPE_oid:
     972           0 :                                 NP_CREATE_BAT(b, oid);
     973             :                                 break;
     974           1 :                         case TYPE_lng:
     975           1 :                                 NP_CREATE_BAT(b, lng);
     976             :                                 break;
     977           0 :                         case TYPE_flt:
     978           0 :                                 NP_CREATE_BAT(b, flt);
     979             :                                 break;
     980          23 :                         case TYPE_dbl:
     981      400146 :                                 NP_CREATE_BAT(b, dbl);
     982             :                                 break;
     983             : #ifdef HAVE_HGE
     984           0 :                         case TYPE_hge:
     985           0 :                                 NP_CREATE_BAT(b, hge);
     986             :                                 break;
     987             : #endif
     988          19 :                         case TYPE_str: {
     989             :                                 bool *mask = NULL;
     990             :                                 char *data = NULL;
     991          19 :                                 char *utf8_string = NULL;
     992          19 :                                 if (ret->mask_data != NULL) {
     993             :                                         mask = (bool *)ret->mask_data;
     994             :                                 }
     995          19 :                                 if (ret->array_data == NULL) {
     996           0 :                                         msg = createException(
     997             :                                                 MAL, "pyapi3.eval",
     998             :                                                 SQLSTATE(PY000) "No return value stored in the structure.  n");
     999           0 :                                         goto wrapup;
    1000             :                                 }
    1001             :                                 data = (char *)ret->array_data;
    1002             : 
    1003          19 :                                 if (ret->result_type != NPY_OBJECT) {
    1004          14 :                                         utf8_string =
    1005          14 :                                                 GDKzalloc(utf8string_minlength + ret->memory_size + 1);
    1006          14 :                                         utf8_string[utf8string_minlength + ret->memory_size] = '\0';
    1007             :                                 }
    1008             : 
    1009          19 :                                 b = COLnew(seqbase, TYPE_str, (BUN)ret->count, TRANSIENT);
    1010          19 :                                 b->tnil = false;
    1011          19 :                                 b->tnonil = true;
    1012          19 :                                 b->tkey = false;
    1013          19 :                                 b->tsorted = false;
    1014          19 :                                 b->trevsorted = false;
    1015     1100099 :                                 NP_INSERT_STRING_BAT(b);
    1016          19 :                                 if (utf8_string)
    1017          19 :                                         GDKfree(utf8_string);
    1018          19 :                                 BATsetcount(b, (BUN)ret->count);
    1019          19 :                                 BATsettrivprop(b);
    1020          19 :                                 break;
    1021             :                         }
    1022           0 :                         default:
    1023           0 :                                 msg = createException(MAL, "pyapi3.eval",
    1024             :                                                                           SQLSTATE(PY000) "Unrecognized BAT type %s.\n",
    1025             :                                                                           BatType_Format(bat_type));
    1026           0 :                                 goto wrapup;
    1027             :                 }
    1028             :         }
    1029         176 :         if (ConvertableSQLType(type)) {
    1030             :                 BAT *result;
    1031           6 :                 msg = ConvertToSQLType(NULL, b, type, &result, &bat_type);
    1032           6 :                 if (msg != MAL_SUCCEED) {
    1033           0 :                         goto wrapup;
    1034             :                 }
    1035           6 :                 b = result;
    1036             :         }
    1037             : 
    1038             :         return b;
    1039             : 
    1040           0 :   wrapup:
    1041           0 :         *return_message = msg;
    1042           0 :         return NULL;
    1043             : }
    1044             : 
    1045         472 : bit ConvertableSQLType(sql_subtype *sql_subtype)
    1046             : {
    1047         472 :         switch (GetSQLType(sql_subtype)) {
    1048             :                 case EC_DATE:
    1049             :                 case EC_TIME:
    1050             :                 case EC_TIMESTAMP:
    1051             :                 case EC_DEC:
    1052             :                         return 1;
    1053             :         }
    1054         464 :         return 0;
    1055             : }
    1056             : 
    1057         648 : int GetSQLType(sql_subtype *sql_subtype)
    1058             : {
    1059         648 :         if (!sql_subtype)
    1060             :                 return -1;
    1061         441 :         if (!sql_subtype->type)
    1062             :                 return -1;
    1063         441 :         return (int) sql_subtype->type->eclass;
    1064             : }
    1065             : 
    1066           7 : str ConvertFromSQLType(BAT *b, sql_subtype *sql_subtype, BAT **ret_bat,
    1067             :                                            int *ret_type)
    1068             : {
    1069             :         str res = MAL_SUCCEED;
    1070             :         int conv_type;
    1071             : 
    1072           7 :         assert(sql_subtype);
    1073           7 :         assert(sql_subtype->type);
    1074             : 
    1075           7 :         switch (sql_subtype->type->eclass) {
    1076             :                 case EC_DATE:
    1077             :                 case EC_TIME:
    1078             :                 case EC_TIMESTAMP:
    1079             :                         conv_type = TYPE_str;
    1080             :                         break;
    1081             :                 case EC_DEC:
    1082             :                         conv_type = TYPE_dbl;
    1083             :                         break;
    1084             :                 default:
    1085             :                         conv_type = TYPE_str;
    1086             :         }
    1087             : 
    1088             :         if (conv_type == TYPE_str) {
    1089             :                 BUN p = 0, q = 0;
    1090           5 :                 char *result = NULL;
    1091           5 :                 size_t length = 0;
    1092           5 :                 ssize_t (*strConversion)(str *, size_t *, const void *, bool) =
    1093           5 :                         BATatoms[b->ttype].atomToStr;
    1094           5 :                 *ret_bat = COLnew(0, TYPE_str, 0, TRANSIENT);
    1095           5 :                 *ret_type = conv_type;
    1096           5 :                 if (!(*ret_bat)) {
    1097           0 :                         return createException(MAL, "pyapi3.eval",
    1098             :                                                                    SQLSTATE(HY013) MAL_MALLOC_FAIL " string conversion BAT.");
    1099             :                 }
    1100           5 :                 BATiter li = bat_iterator(b);
    1101          11 :                 BATloop(b, p, q)
    1102             :                 {
    1103           6 :                         void *element = (void *)BUNtail(li, p);
    1104           6 :                         if (strConversion(&result, &length, element, false) < 0) {
    1105           0 :                                 bat_iterator_end(&li);
    1106           0 :                                 BBPunfix((*ret_bat)->batCacheid);
    1107           0 :                                 return createException(MAL, "pyapi3.eval",
    1108             :                                                                            SQLSTATE(PY000) "Failed to convert element to string.");
    1109             :                         }
    1110           6 :                         if (BUNappend(*ret_bat, result, false) != GDK_SUCCEED) {
    1111           0 :                                 bat_iterator_end(&li);
    1112           0 :                                 BBPunfix((*ret_bat)->batCacheid);
    1113           0 :                                 throw(MAL, "pyapi3.eval", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1114             :                         }
    1115             :                 }
    1116           5 :                 bat_iterator_end(&li);
    1117           5 :                 if (result) {
    1118           5 :                         GDKfree(result);
    1119             :                 }
    1120           5 :                 return res;
    1121             :         } else if (conv_type == TYPE_dbl) {
    1122           2 :                 int bat_type = ATOMstorage(b->ttype);
    1123           2 :                 int hpos = sql_subtype->scale;
    1124           2 :                 bat result = 0;
    1125             :                 // decimal values can be stored in various numeric fields, so check the
    1126             :                 // numeric field and convert the one it's actually stored in
    1127           2 :                 switch (bat_type) {
    1128           0 :                         case TYPE_bte:
    1129           0 :                                 res = batbte_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
    1130           0 :                                 break;
    1131           0 :                         case TYPE_sht:
    1132           0 :                                 res = batsht_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
    1133           0 :                                 break;
    1134           0 :                         case TYPE_int:
    1135           0 :                                 res = batint_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
    1136           0 :                                 break;
    1137           2 :                         case TYPE_lng:
    1138           2 :                                 res = batlng_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
    1139           2 :                                 break;
    1140             : #ifdef HAVE_HGE
    1141           0 :                         case TYPE_hge:
    1142           0 :                                 res = bathge_dec2_dbl(&result, &hpos, &b->batCacheid, NULL);
    1143           0 :                                 break;
    1144             : #endif
    1145           0 :                         default:
    1146           0 :                                 return createException(MAL, "pyapi3.eval",
    1147             :                                                                            "Unsupported decimal storage type.");
    1148             :                 }
    1149           2 :                 if (res == MAL_SUCCEED) {
    1150           2 :                         *ret_bat = BATdescriptor(result);
    1151           2 :                         *ret_type = TYPE_dbl;
    1152             :                 } else {
    1153           0 :                         *ret_bat = NULL;
    1154             :                 }
    1155           2 :                 return res;
    1156             :         }
    1157             :         return createException(MAL, "pyapi3.eval", "Unrecognized conv type.");
    1158             : }
    1159             : 
    1160           6 : str ConvertToSQLType(Client cntxt, BAT *b, sql_subtype *sql_subtype,
    1161             :                                          BAT **ret_bat, int *ret_type)
    1162             : {
    1163             :         str res = MAL_SUCCEED;
    1164           6 :         bat result_bat = 0;
    1165           6 :         int digits = sql_subtype->digits;
    1166           6 :         int scale = sql_subtype->scale;
    1167             :         (void)cntxt;
    1168             : 
    1169             :         assert(sql_subtype);
    1170           6 :         assert(sql_subtype->type);
    1171             : 
    1172           6 :         switch (sql_subtype->type->eclass) {
    1173           1 :                 case EC_TIMESTAMP:
    1174           1 :                         res = batstr_2time_timestamp(&result_bat, &b->batCacheid, NULL, &digits);
    1175           1 :                         break;
    1176           1 :                 case EC_TIME:
    1177           1 :                         res = batstr_2time_daytime(&result_bat, &b->batCacheid, NULL, &digits);
    1178           1 :                         break;
    1179           1 :                 case EC_DATE:
    1180           1 :                         res = batstr_2_date(&result_bat, &b->batCacheid, NULL);
    1181           1 :                         break;
    1182           3 :                 case EC_DEC:
    1183           3 :                         res = batdbl_num2dec_lng(&result_bat, &b->batCacheid, NULL,
    1184             :                                                                          &digits, &scale);
    1185           3 :                         break;
    1186           0 :                 default:
    1187           0 :                         return createException(
    1188             :                                 MAL, "pyapi3.eval",
    1189             :                                 "Convert To SQL Type: Unrecognized SQL type %s (%d).",
    1190             :                                 sql_subtype->type->base.name, (int) sql_subtype->type->eclass);
    1191             :         }
    1192           6 :         if (res == MAL_SUCCEED) {
    1193           6 :                 *ret_bat = BATdescriptor(result_bat);
    1194           6 :                 *ret_type = (*ret_bat)->ttype;
    1195             :         }
    1196             : 
    1197             :         return res;
    1198             : }
    1199             : 
    1200         224 : ssize_t PyType_Size(PyObject *obj)
    1201             : {
    1202         224 :         if (PyType_IsPyScalar(obj)) {
    1203             :                 return 1;
    1204             :         }
    1205          62 :         if (PyArray_Check(obj)) {
    1206          57 :                 return PyArray_Size(obj);
    1207             :         }
    1208           5 :         if (PyList_Check(obj)) {
    1209           5 :                 return Py_SIZE(obj);
    1210             :         }
    1211             :         return -1;
    1212             : }
    1213             : 
    1214         301 : bit IsStandardBATType(int type)
    1215             : {
    1216         301 :         switch (type) {
    1217             :                 case TYPE_bit:
    1218             :                 case TYPE_bte:
    1219             :                 case TYPE_sht:
    1220             :                 case TYPE_int:
    1221             :                 case TYPE_oid:
    1222             :                 case TYPE_lng:
    1223             :                 case TYPE_flt:
    1224             :                 case TYPE_dbl:
    1225             : #ifdef HAVE_HGE
    1226             :                 case TYPE_hge:
    1227             : #endif
    1228             :                 case TYPE_str:
    1229             :                         return 1;
    1230           5 :                 default:
    1231           5 :                         return 0;
    1232             :         }
    1233             : }
    1234             : 
    1235           8 : static void conversion_import_array(void) { _import_array(); }
    1236             : 
    1237           8 : str _conversion_init(void)
    1238             : {
    1239             :         str msg = MAL_SUCCEED;
    1240             :         conversion_import_array();
    1241             : 
    1242           8 :         return msg;
    1243             : }

Generated by: LCOV version 1.14