LCOV - code coverage report
Current view: top level - sql/backends/monet5/UDF/pyapi3 - convert_loops.h (source / functions) Hit Total Coverage
Test: coverage.info Lines: 3 12 25.0 %
Date: 2020-06-29 20:00:14 Functions: 1 1 100.0 %

          Line data    Source code
       1             : /*
       2             :  * This Source Code Form is subject to the terms of the Mozilla Public
       3             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       4             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       5             :  *
       6             :  * Copyright 1997 - July 2008 CWI, August 2008 - 2020 MonetDB B.V.
       7             :  */
       8             : 
       9             : /*
      10             :  * M. Raasveldt
      11             :  * Conversion loops used to convert from BAT <> NumPy Array
      12             :  * these are in a separate header because they are used in multiple places
      13             :  */
      14             : 
      15             : #define BAT_TO_NP_CREATE_ALWAYS(bat, nptpe)                                    \
      16             :         do {                                                                       \
      17             :                 vararray = PyArray_Arange(0, (double)bat->batCount, 1, nptpe);         \
      18             :         } while(0)                                                                 \
      19             : 
      20             : #define BAT_TO_NP(bat, mtpe, nptpe)                                            \
      21             :         do {                                                                       \
      22             :                 if (copy) {                                                            \
      23             :                         vararray = PyArray_EMPTY(1, elements, nptpe, 0);                   \
      24             :                         memcpy(PyArray_DATA((PyArrayObject *)vararray), Tloc(bat, 0),      \
      25             :                                 sizeof(mtpe) * (t_end - t_start));                             \
      26             :                 } else {                                                               \
      27             :                         vararray =                                                         \
      28             :                                 PyArray_New(&PyArray_Type, 1, elements, nptpe, NULL,           \
      29             :                                                         &((mtpe *)Tloc(bat, 0))[t_start], 0,               \
      30             :                                                         NPY_ARRAY_CARRAY || !NPY_ARRAY_WRITEABLE, NULL);   \
      31             :                 }                                                                      \
      32             :         } while(0)                                                                 \
      33             : 
      34             : // This #define creates a new BAT with the internal data and mask from a Numpy
      35             : // array, without copying the data
      36             : // 'bat' is a BAT* pointer, which will contain the new BAT. TYPE_'mtpe' is the
      37             : // BAT type, and 'batstore' is the heap storage type of the BAT (this should be
      38             : // STORE_CMEM or STORE_SHARED)
      39             : #define nancheck_flt(bat)                                                      \
      40             :         do {                                                                       \
      41             :                 for (iu = 0; iu < ret->count; iu++) {                                  \
      42             :                         if (isnan(((flt *)data)[index_offset * ret->count + iu])) {        \
      43             :                                 ((flt *)data)[index_offset * ret->count + iu] = flt_nil;       \
      44             :                                 bat->tnil = true;                                              \
      45             :                         }                                                                  \
      46             :                 }                                                                      \
      47             :                 bat->tnonil = !bat->tnil;                                              \
      48             :         } while (0)
      49             : #define nancheck_dbl(bat)                                                      \
      50             :         do {                                                                       \
      51             :                 for (iu = 0; iu < ret->count; iu++) {                                  \
      52             :                         if (isnan(((dbl *)data)[index_offset * ret->count + iu])) {        \
      53             :                                 ((dbl *)data)[index_offset * ret->count + iu] = dbl_nil;       \
      54             :                                 bat->tnil = true;                                              \
      55             :                         }                                                                  \
      56             :                 }                                                                      \
      57             :                 bat->tnonil = !bat->tnil;                                              \
      58             :         } while (0)
      59             : #define nancheck_bit(bat) ((void)0)
      60             : #define nancheck_bte(bat) ((void)0)
      61             : #define nancheck_sht(bat) ((void)0)
      62             : #define nancheck_int(bat) ((void)0)
      63             : #define nancheck_lng(bat) ((void)0)
      64             : #define nancheck_hge(bat) ((void)0) /* not used if no HAVE_HGE */
      65             : #define nancheck_oid(bat) ((void)0)
      66             : #if defined(HAVE_FORK) && !defined(HAVE_EMBEDDED)
      67             : #define CREATE_BAT_ZEROCOPY(bat, mtpe, batstore)                               \
      68             :         {                                                                          \
      69             :                 bat = COLnew(seqbase, TYPE_##mtpe, 0, TRANSIENT);                      \
      70             :                 if (bat == NULL) {                                                     \
      71             :                 msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Cannot create column");     \
      72             :                         goto wrapup;                                                       \
      73             :                 }                                                                      \
      74             :                 bat->tnil = false;                                                     \
      75             :                 bat->tnonil = true;                                                    \
      76             :                 bat->tkey = false;                                                     \
      77             :                 bat->tsorted = false;                                                  \
      78             :                 bat->trevsorted = false;                                               \
      79             :                 /*Change nil values to the proper values, if they exist*/              \
      80             :                 if (mask != NULL) {                                                    \
      81             :                         for (iu = 0; iu < ret->count; iu++) {                              \
      82             :                                 if (mask[index_offset * ret->count + iu] == TRUE) {            \
      83             :                                         (*(mtpe *)(&data[(index_offset * ret->count + iu) *        \
      84             :                                                                          ret->memory_size])) = mtpe##_nil;         \
      85             :                                         bat->tnil = true;                                          \
      86             :                                 }                                                              \
      87             :                         }                                                                  \
      88             :                         bat->tnonil = !bat->tnil;                                          \
      89             :                 } else {                                                               \
      90             :                         bat->tnil = false;                                                 \
      91             :                         bat->tnonil = false;                                               \
      92             :                         nancheck_##mtpe(bat);                                              \
      93             :                 }                                                                      \
      94             :                                                                                \
      95             :                 /*When we create a BAT a small part of memory is allocated, free it*/  \
      96             :                 GDKfree(bat->theap.base);                                              \
      97             :                 bat->theap.base =                                                      \
      98             :                         &data[(index_offset * ret->count) * ret->memory_size];             \
      99             :                 bat->theap.size = ret->count * ret->memory_size;                       \
     100             :                 bat->theap.free =                                                      \
     101             :                         bat->theap.size; /*There are no free places in the array*/         \
     102             :                 /*If index_offset > 0, we are mapping part of a multidimensional       \
     103             :                  * array.*/                                                            \
     104             :                 /*The entire array will be cleared when the part with index_offset=0   \
     105             :                  * is freed*/                                                          \
     106             :                 /*So we set this part of the mapping to 'NOWN'*/                       \
     107             :                 if (index_offset > 0)                                                  \
     108             :                         bat->theap.storage = STORE_NOWN;                                   \
     109             :                 else {                                                                 \
     110             :                         bat->theap.storage = batstore;                                     \
     111             :                         if (batstore == STORE_MMAPABS) {                                   \
     112             :                                 /* If we are taking data from a MMAP file, set the filename to \
     113             :                                  * the absolute path */                                        \
     114             :                                 char address[100];                                             \
     115             :                                 GDKmmapfile(address, sizeof(address), ret->mmap_id);           \
     116             :                                 snprintf(bat->theap.filename, sizeof(bat->theap.filename),     \
     117             :                                         "%s%c%s.tmp", BATDIR, DIR_SEP, address);                   \
     118             :                                 ret->mmap_id = -1;                                             \
     119             :                         }                                                                  \
     120             :                 }                                                                      \
     121             :                 bat->theap.newstorage = STORE_MEM;                                     \
     122             :                 bat->batCount = ret->count;                                            \
     123             :                 bat->batCapacity = ret->count;                                         \
     124             :                 bat->batCopiedtodisk = false;                                          \
     125             :                 /*Take over the data from the numpy array*/                            \
     126             :                 if (ret->numpy_array != NULL)                                          \
     127             :                         PyArray_CLEARFLAGS((PyArrayObject *)ret->numpy_array,              \
     128             :                                                            NPY_ARRAY_OWNDATA);                             \
     129             :         }
     130             : #else
     131             : #define CREATE_BAT_ZEROCOPY(bat, mtpe, batstore)                               \
     132             :         {                                                                          \
     133             :                 bat = COLnew(seqbase, TYPE_##mtpe, 0, TRANSIENT);                      \
     134             :                 if (bat == NULL) {                                                     \
     135             :                         msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Cannot create column");     \
     136             :                         goto wrapup;                                                       \
     137             :                 }                                                                      \
     138             :                 bat->tnil = false;                                                     \
     139             :                 bat->tnonil = true;                                                    \
     140             :                 bat->tkey = false;                                                     \
     141             :                 bat->tsorted = false;                                                  \
     142             :                 bat->trevsorted = false;                                               \
     143             :                 /*Change nil values to the proper values, if they exist*/              \
     144             :                 if (mask != NULL) {                                                    \
     145             :                         for (iu = 0; iu < ret->count; iu++) {                              \
     146             :                                 if (mask[index_offset * ret->count + iu] == TRUE) {            \
     147             :                                         (*(mtpe *)(&data[(index_offset * ret->count + iu) *        \
     148             :                                                                          ret->memory_size])) = mtpe##_nil;         \
     149             :                                         bat->tnil = true;                                          \
     150             :                                 }                                                              \
     151             :                         }                                                                  \
     152             :                         bat->tnonil = !bat->tnil;                                          \
     153             :                 } else {                                                               \
     154             :                         bat->tnil = false;                                                 \
     155             :                         bat->tnonil = false;                                               \
     156             :                         nancheck_##mtpe(bat);                                              \
     157             :                 }                                                                      \
     158             :                 /*When we create a BAT a small part of memory is allocated, free it*/  \
     159             :                 GDKfree(bat->theap.base);                                              \
     160             :                 bat->theap.base =                                                      \
     161             :                         &data[(index_offset * ret->count) * ret->memory_size];             \
     162             :                 bat->theap.size = ret->count * ret->memory_size;                       \
     163             :                 bat->theap.free =                                                      \
     164             :                         bat->theap.size; /*There are no free places in the array*/         \
     165             :                 /*If index_offset > 0, we are mapping part of a multidimensional       \
     166             :                  * array.*/                                                            \
     167             :                 /*The entire array will be cleared when the part with index_offset=0   \
     168             :                  * is freed*/                                                          \
     169             :                 /*So we set this part of the mapping to 'NOWN'*/                       \
     170             :                 if (index_offset > 0)                                                  \
     171             :                         bat->theap.storage = STORE_NOWN;                                   \
     172             :                 else {                                                                 \
     173             :                         bat->theap.storage = batstore;                                     \
     174             :                 }                                                                      \
     175             :                 bat->theap.newstorage = STORE_MEM;                                     \
     176             :                 bat->batCount = (BUN)ret->count;                                       \
     177             :                 bat->batCapacity = (BUN)ret->count;                                    \
     178             :                 bat->batCopiedtodisk = false;                                          \
     179             :                 /*Take over the data from the numpy array*/                            \
     180             :                 if (ret->numpy_array != NULL)                                          \
     181             :                         PyArray_CLEARFLAGS((PyArrayObject *)ret->numpy_array,              \
     182             :                                                            NPY_ARRAY_OWNDATA);                             \
     183             :         }
     184             : #endif
     185             : 
     186             : // This #define converts a Numpy Array to a BAT by copying the internal data to
     187             : // the BAT. It assumes the BAT 'bat' is already created with the proper size.
     188             : // This should only be used with integer data that can be cast. It assumes the
     189             : // Numpy Array has an internal array of type 'mtpe_from', and the BAT has an
     190             : // internal array of type 'mtpe_to'.
     191             : // it then does the cast by simply doing BAT[i] = (mtpe_to)
     192             : // ((mtpe_from*)NUMPY_ARRAY[i]), which only works if both mtpe_to and mtpe_from
     193             : // are integers
     194             : #define NP_COL_BAT_LOOP(bat, mtpe_to, mtpe_from, index)                        \
     195             :         {                                                                          \
     196             :                 if (mask == NULL) {                                                    \
     197             :                         for (iu = 0; iu < ret->count; iu++) {                              \
     198             :                                 ((mtpe_to *)Tloc(bat, 0))[index + iu] = (mtpe_to)(             \
     199             :                                         *(mtpe_from *)(&data[(index_offset * ret->count + iu) *    \
     200             :                                                                                  ret->memory_size]));                  \
     201             :                         }                                                                  \
     202             :                 } else {                                                               \
     203             :                         for (iu = 0; iu < ret->count; iu++) {                              \
     204             :                                 if (mask[index_offset * ret->count + iu] == TRUE) {            \
     205             :                                         bat->tnil = true;                                          \
     206             :                                         ((mtpe_to *)Tloc(bat, 0))[index + iu] = mtpe_to##_nil;     \
     207             :                                 } else {                                                       \
     208             :                                         ((mtpe_to *)Tloc(bat, 0))[index + iu] = (mtpe_to)(*(       \
     209             :                                                 mtpe_from *)(&data[(index_offset * ret->count + iu) *  \
     210             :                                                                                    ret->memory_size]));                \
     211             :                                 }                                                              \
     212             :                         }                                                                  \
     213             :                 }                                                                      \
     214             :         }
     215             : #define NP_COL_BAT_LOOPF(bat, mtpe_to, mtpe_from, index)                       \
     216             :         {                                                                          \
     217             :                 if (mask == NULL) {                                                    \
     218             :                         for (iu = 0; iu < ret->count; iu++) {                              \
     219             :                                 if (isnan((                                                    \
     220             :                                                 (mtpe_from *)data)[index_offset * ret->count + iu])) { \
     221             :                                         bat->tnil = true;                                          \
     222             :                                         ((mtpe_to *)Tloc(bat, 0))[index + iu] = mtpe_to##_nil;     \
     223             :                                 } else {                                                       \
     224             :                                         ((mtpe_to *)Tloc(bat, 0))[index + iu] = (mtpe_to)(         \
     225             :                                                 (mtpe_from *)data)[index_offset * ret->count + iu];    \
     226             :                                 }                                                              \
     227             :                         }                                                                  \
     228             :                 } else {                                                               \
     229             :                         for (iu = 0; iu < ret->count; iu++) {                              \
     230             :                                 if (mask[index_offset * ret->count + iu] == TRUE ||            \
     231             :                                         isnan((                                                    \
     232             :                                                 (mtpe_from *)data)[index_offset * ret->count + iu])) { \
     233             :                                         bat->tnil = true;                                          \
     234             :                                         ((mtpe_to *)Tloc(bat, 0))[index + iu] = mtpe_to##_nil;     \
     235             :                                 } else {                                                       \
     236             :                                         ((mtpe_to *)Tloc(bat, 0))[index + iu] = (mtpe_to)(*(       \
     237             :                                                 mtpe_from *)(&data[(index_offset * ret->count + iu) *  \
     238             :                                                                                    ret->memory_size]));                \
     239             :                                 }                                                              \
     240             :                         }                                                                  \
     241             :                 }                                                                      \
     242             :         }
     243             : 
     244             : // This #define converts a Numpy Array to a BAT by copying the internal data to
     245             : // the BAT. It converts the data from the Numpy Array to the BAT using a
     246             : // function
     247             : // This function has to have the prototype 'bool function(void *data, size_t
     248             : // memory_size, mtpe_to *resulting_value)', and either return False (if
     249             : // conversion fails)
     250             : //  or write the value into the 'resulting_value' pointer. This is used
     251             : //  convertring strings/unicodes/python objects to numeric values.
     252             : #define NP_COL_BAT_LOOP_FUNC(bat, mtpe_to, func, ptrtpe, index)                \
     253             :         {                                                                          \
     254             :                 mtpe_to value;                                                         \
     255             :                 if (mask == NULL) {                                                    \
     256             :                         for (iu = 0; iu < ret->count; iu++) {                              \
     257             :                                 msg = func((ptrtpe *)&data[(index_offset * ret->count + iu) *  \
     258             :                                                                                    ret->memory_size],                  \
     259             :                                                    ret->memory_size, &value);                          \
     260             :                                 if (msg != MAL_SUCCEED) {                                      \
     261             :                                         goto wrapup;                                               \
     262             :                                 }                                                              \
     263             :                                 ((mtpe_to *)Tloc(bat, 0))[index + iu] = value;                 \
     264             :                                 if (!bat->tnil)                                                \
     265             :                                         bat->tnil = is_##mtpe_to##_nil(value);                     \
     266             :                         }                                                                  \
     267             :                 } else {                                                               \
     268             :                         for (iu = 0; iu < ret->count; iu++) {                              \
     269             :                                 if (mask[index_offset * ret->count + iu] == TRUE) {            \
     270             :                                         bat->tnil = true;                                          \
     271             :                                         ((mtpe_to *)Tloc(bat, 0))[index + iu] = mtpe_to##_nil;     \
     272             :                                 } else {                                                       \
     273             :                                         msg = func(                                                \
     274             :                                                 (ptrtpe *)&data[(index_offset * ret->count + iu) *     \
     275             :                                                                                 ret->memory_size],                     \
     276             :                                                 ret->memory_size, &value);                             \
     277             :                                         if (msg != MAL_SUCCEED) {                                  \
     278             :                                                 goto wrapup;                                           \
     279             :                                         }                                                          \
     280             :                                         ((mtpe_to *)Tloc(bat, 0))[index + iu] = value;             \
     281             :                                         if (!bat->tnil)                                            \
     282             :                                                 bat->tnil = is_##mtpe_to##_nil(value);                 \
     283             :                                 }                                                              \
     284             :                         }                                                                  \
     285             :                 }                                                                      \
     286             :         }
     287             : 
     288             : 
     289             : static gdk_return
     290     1100090 : convert_and_append(BAT* b, const char* text, bool force) {
     291     1100090 :         if (b->ttype == TYPE_str) {
     292     1100090 :                 return BUNappend(b, text, force);
     293           0 :         } else if (text == str_nil) {
     294           0 :                 return BUNappend(b, BATatoms[b->ttype].atomNull, force);
     295             :         } else {
     296           0 :                 void* element = NULL;
     297           0 :                 size_t len = 0;
     298           0 :                 gdk_return ret;
     299             : 
     300           0 :                 if (BATatoms[b->ttype].atomFromStr(text, &len, &element, false) < 0)
     301             :                         return GDK_FAIL;
     302           0 :                 ret = BUNappend(b, element, force);
     303           0 :                 GDKfree(element);
     304           0 :                 return ret;
     305             :         }
     306             : }
     307             : 
     308             : // This #define is for converting a numeric numpy array into a string BAT.
     309             : // 'conv' is a function that turns a numeric value of type 'mtpe' to a char*
     310             : // array.
     311             : #define NP_COL_BAT_STR_LOOP(bat, mtpe, fmt)                                    \
     312             :         if (mask == NULL) {                                                        \
     313             :                 for (iu = 0; iu < ret->count; iu++) {                                  \
     314             :                         snprintf(utf8_string, utf8string_minlength, fmt,                   \
     315             :                                          *((mtpe *)&data[(index_offset * ret->count + iu) *        \
     316             :                                                                          ret->memory_size]));                      \
     317             :                         if (convert_and_append(bat, utf8_string, false) != GDK_SUCCEED) {           \
     318             :                                 msg =                                                          \
     319             :                                         createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "BUNappend failed.\n"); \
     320             :                                 goto wrapup;                                                   \
     321             :                         }                                                                  \
     322             :                 }                                                                      \
     323             :         } else {                                                                   \
     324             :                 for (iu = 0; iu < ret->count; iu++) {                                  \
     325             :                         if (mask[index_offset * ret->count + iu] == TRUE) {                \
     326             :                                 bat->tnil = true;                                              \
     327             :                                 if (convert_and_append(bat, str_nil, false) != GDK_SUCCEED) {  \
     328             :                                         msg = createException(MAL, "pyapi3.eval",                  \
     329             :                                                                                   SQLSTATE(PY000) "BUNappend failed.\n");              \
     330             :                                         goto wrapup;                                               \
     331             :                                 }                                                              \
     332             :                         } else {                                                           \
     333             :                                 snprintf(utf8_string, utf8string_minlength, fmt,               \
     334             :                                                  *((mtpe *)&data[(index_offset * ret->count + iu) *    \
     335             :                                                                                  ret->memory_size]));                  \
     336             :                                 if (convert_and_append(bat, utf8_string, false) != GDK_SUCCEED) {       \
     337             :                                         msg = createException(MAL, "pyapi3.eval",                  \
     338             :                                                                                   SQLSTATE(PY000) "BUNappend failed.\n");              \
     339             :                                         goto wrapup;                                               \
     340             :                                 }                                                              \
     341             :                         }                                                                  \
     342             :                 }                                                                      \
     343             :         }
     344             : 
     345             : #define NP_INSERT_BAT(bat, mtpe, index)                                        \
     346             :         {                                                                          \
     347             :                 switch (ret->result_type) {                                            \
     348             :                         case NPY_BOOL:                                                     \
     349             :                                 NP_COL_BAT_LOOP(bat, mtpe, char, index);                       \
     350             :                                 break;                                                         \
     351             :                         case NPY_BYTE:                                                     \
     352             :                                 NP_COL_BAT_LOOP(bat, mtpe, char, index);                       \
     353             :                                 break;                                                         \
     354             :                         case NPY_SHORT:                                                    \
     355             :                                 NP_COL_BAT_LOOP(bat, mtpe, short, index);                      \
     356             :                                 break;                                                         \
     357             :                         case NPY_INT:                                                      \
     358             :                                 NP_COL_BAT_LOOP(bat, mtpe, int, index);                        \
     359             :                                 break;                                                         \
     360             :                         case NPY_LONG:                                                     \
     361             :                                 NP_COL_BAT_LOOP(bat, mtpe, long, index);                       \
     362             :                                 break;                                                         \
     363             :                         case NPY_LONGLONG:                                                 \
     364             :                                 NP_COL_BAT_LOOP(bat, mtpe, long long, index);                  \
     365             :                                 break;                                                         \
     366             :                         case NPY_UBYTE:                                                    \
     367             :                                 NP_COL_BAT_LOOP(bat, mtpe, unsigned char, index);              \
     368             :                                 break;                                                         \
     369             :                         case NPY_USHORT:                                                   \
     370             :                                 NP_COL_BAT_LOOP(bat, mtpe, unsigned short, index);             \
     371             :                                 break;                                                         \
     372             :                         case NPY_UINT:                                                     \
     373             :                                 NP_COL_BAT_LOOP(bat, mtpe, unsigned int, index);               \
     374             :                                 break;                                                         \
     375             :                         case NPY_ULONG:                                                    \
     376             :                                 NP_COL_BAT_LOOP(bat, mtpe, unsigned long, index);              \
     377             :                                 break;                                                         \
     378             :                         case NPY_ULONGLONG:                                                \
     379             :                                 NP_COL_BAT_LOOP(bat, mtpe, unsigned long long, index);         \
     380             :                                 break;                                                         \
     381             :                         case NPY_FLOAT16:                                                  \
     382             :                         case NPY_FLOAT:                                                    \
     383             :                                 NP_COL_BAT_LOOPF(bat, mtpe, float, index);                     \
     384             :                                 break;                                                         \
     385             :                         case NPY_DOUBLE:                                                   \
     386             :                                 NP_COL_BAT_LOOPF(bat, mtpe, double, index);                    \
     387             :                                 break;                                                         \
     388             :                         case NPY_LONGDOUBLE:                                               \
     389             :                                 NP_COL_BAT_LOOPF(bat, mtpe, long double, index);               \
     390             :                                 break;                                                         \
     391             :                         case NPY_STRING:                                                   \
     392             :                                 NP_COL_BAT_LOOP_FUNC(bat, mtpe, str_to_##mtpe, char, index);   \
     393             :                                 break;                                                         \
     394             :                         case NPY_UNICODE:                                                  \
     395             :                                 NP_COL_BAT_LOOP_FUNC(bat, mtpe, unicode_to_##mtpe,             \
     396             :                                                                          PythonUnicodeType, index);                \
     397             :                                 break;                                                         \
     398             :                         case NPY_OBJECT:                                                   \
     399             :                                 NP_COL_BAT_LOOP_FUNC(bat, mtpe, pyobject_to_##mtpe,            \
     400             :                                                                          PyObject *, index);                       \
     401             :                                 break;                                                         \
     402             :                         default:                                                           \
     403             :                                 msg = createException(                                         \
     404             :                                         MAL, "pyapi3.eval",                                        \
     405             :                                         SQLSTATE(PY000) "Unrecognized type. Could not convert to %s.\n",           \
     406             :                                         BatType_Format(TYPE_##mtpe));                              \
     407             :                                 goto wrapup;                                                   \
     408             :                 }                                                                      \
     409             :                 bat->tnonil = !bat->tnil;                                              \
     410             :         }
     411             : 
     412             : #define NP_INSERT_STRING_BAT(b)                                                \
     413             :         switch (ret->result_type) {                                                \
     414             :                 case NPY_BOOL:                                                         \
     415             :                         NP_COL_BAT_STR_LOOP(b, bit, "%hhd");                               \
     416             :                         break;                                                             \
     417             :                 case NPY_BYTE:                                                         \
     418             :                         NP_COL_BAT_STR_LOOP(b, bte, "%hhd");                               \
     419             :                         break;                                                             \
     420             :                 case NPY_SHORT:                                                        \
     421             :                         NP_COL_BAT_STR_LOOP(b, sht, "%hd");                                \
     422             :                         break;                                                             \
     423             :                 case NPY_INT:                                                          \
     424             :                         NP_COL_BAT_STR_LOOP(b, int, "%d");                                 \
     425             :                         break;                                                             \
     426             :                 case NPY_LONG:                                                         \
     427             :                         NP_COL_BAT_STR_LOOP(b, long, "%ld");                               \
     428             :                         break;                                                             \
     429             :                 case NPY_LONGLONG:                                                     \
     430             :                         NP_COL_BAT_STR_LOOP(b, lng, LLFMT);                                \
     431             :                         break;                                                             \
     432             :                 case NPY_UBYTE:                                                        \
     433             :                         NP_COL_BAT_STR_LOOP(b, unsigned char, "%hhu");                     \
     434             :                         break;                                                             \
     435             :                 case NPY_USHORT:                                                       \
     436             :                         NP_COL_BAT_STR_LOOP(b, unsigned short, "%hu");                     \
     437             :                         break;                                                             \
     438             :                 case NPY_UINT:                                                         \
     439             :                         NP_COL_BAT_STR_LOOP(b, unsigned int, "%u");                        \
     440             :                         break;                                                             \
     441             :                 case NPY_ULONG:                                                        \
     442             :                         NP_COL_BAT_STR_LOOP(b, unsigned long, "%lu");                      \
     443             :                         break;                                                             \
     444             :                 case NPY_ULONGLONG:                                                    \
     445             :                         NP_COL_BAT_STR_LOOP(b, ulng, ULLFMT);                              \
     446             :                         break;                                                             \
     447             :                 case NPY_FLOAT16:                                                      \
     448             :                 case NPY_FLOAT:                                                        \
     449             :                         NP_COL_BAT_STR_LOOP(b, flt, "%f");                                 \
     450             :                         break;                                                             \
     451             :                 case NPY_DOUBLE:                                                       \
     452             :                 case NPY_LONGDOUBLE:                                                   \
     453             :                         NP_COL_BAT_STR_LOOP(b, dbl, "%lf");                                \
     454             :                         break;                                                             \
     455             :                 case NPY_STRING:                                                       \
     456             :                         for (iu = 0; iu < ret->count; iu++) {                              \
     457             :                                 if (mask != NULL &&                                            \
     458             :                                         (mask[index_offset * ret->count + iu]) == TRUE) {          \
     459             :                                         b->tnil = true;                                            \
     460             :                                         if (convert_and_append(b, str_nil, false) != GDK_SUCCEED) {         \
     461             :                                                 msg = createException(MAL, "pyapi3.eval",              \
     462             :                                                                                           SQLSTATE(PY000) "BUNappend failed.\n");          \
     463             :                                                 goto wrapup;                                           \
     464             :                                         }                                                          \
     465             :                                 } else {                                                       \
     466             :                                         if (!string_copy(&data[(index_offset * ret->count + iu) *  \
     467             :                                                                                    ret->memory_size],                  \
     468             :                                                                          utf8_string, ret->memory_size, false)) {  \
     469             :                                                 msg = createException(MAL, "pyapi3.eval",              \
     470             :                                                                                           SQLSTATE(PY000) "Invalid string encoding used. " \
     471             :                                                                                           "Please return a regular ASCII " \
     472             :                                                                                           "string, or a Numpy_Unicode "    \
     473             :                                                                                           "object.\n");                    \
     474             :                                                 goto wrapup;                                           \
     475             :                                         }                                                          \
     476             :                                         if (convert_and_append(b, utf8_string, false) != GDK_SUCCEED) {     \
     477             :                                                 msg = createException(MAL, "pyapi3.eval",              \
     478             :                                                                                           SQLSTATE(PY000) "BUNappend failed.\n");          \
     479             :                                                 goto wrapup;                                           \
     480             :                                         }                                                          \
     481             :                                 }                                                              \
     482             :                         }                                                                  \
     483             :                         break;                                                             \
     484             :                 case NPY_UNICODE:                                                      \
     485             :                         for (iu = 0; iu < ret->count; iu++) {                              \
     486             :                                 if (mask != NULL &&                                            \
     487             :                                         (mask[index_offset * ret->count + iu]) == TRUE) {          \
     488             :                                         b->tnil = true;                                            \
     489             :                                         if (convert_and_append(b, str_nil, false) != GDK_SUCCEED) {         \
     490             :                                                 msg = createException(MAL, "pyapi3.eval",              \
     491             :                                                                                           SQLSTATE(PY000) "BUNappend failed.\n");          \
     492             :                                                 goto wrapup;                                           \
     493             :                                         }                                                          \
     494             :                                 } else {                                                       \
     495             :                                         utf32_to_utf8(                                             \
     496             :                                                 0, ret->memory_size / 4, utf8_string,                  \
     497             :                                                 (const Py_UNICODE                                      \
     498             :                                                          *)(&data[(index_offset * ret->count + iu) *       \
     499             :                                                                           ret->memory_size]));                     \
     500             :                                         if (convert_and_append(b, utf8_string, false) != GDK_SUCCEED) {     \
     501             :                                                 msg = createException(MAL, "pyapi3.eval",              \
     502             :                                                                                           SQLSTATE(PY000) "BUNappend failed.\n");          \
     503             :                                                 goto wrapup;                                           \
     504             :                                         }                                                          \
     505             :                                 }                                                              \
     506             :                         }                                                                  \
     507             :                         break;                                                             \
     508             :                 case NPY_OBJECT: {                                                     \
     509             :                         /* The resulting array is an array of pointers to various python   \
     510             :                          * objects */                                                      \
     511             :                         /* Because the python objects can be of any size, we need to       \
     512             :                          * allocate a different size utf8_string for every object */       \
     513             :                         /* we will first loop over all the objects to get the maximum size \
     514             :                          * needed, so we only need to do one allocation */                 \
     515             :                         size_t utf8_size = utf8string_minlength;                           \
     516             :                         for (iu = 0; iu < ret->count; iu++) {                              \
     517             :                                 size_t size = utf8string_minlength;                            \
     518             :                                 PyObject *obj;                                                 \
     519             :                                 if (mask != NULL &&                                            \
     520             :                                         (mask[index_offset * ret->count + iu]) == TRUE)            \
     521             :                                         continue;                                                  \
     522             :                                 obj = *((PyObject **)&data[(index_offset * ret->count + iu) *  \
     523             :                                                                                    ret->memory_size]);                 \
     524             :                                 size = pyobject_get_size(obj);                                 \
     525             :                                 if (size > utf8_size)                                          \
     526             :                                         utf8_size = size;                                          \
     527             :                         }                                                                  \
     528             :                         utf8_string = GDKzalloc(utf8_size);                                \
     529             :                         if (utf8_string == NULL) {                      \
     530             :                                 msg = createException(MAL, "pyapi3.eval", SQLSTATE(HY013) MAL_MALLOC_FAIL); \
     531             :                                 goto wrapup;                            \
     532             :                         }                                               \
     533             :                         for (iu = 0; iu < ret->count; iu++) {                              \
     534             :                                 if (mask != NULL &&                                            \
     535             :                                         (mask[index_offset * ret->count + iu]) == TRUE) {          \
     536             :                                         b->tnil = true;                                            \
     537             :                                         if (convert_and_append(b, str_nil, false) != GDK_SUCCEED) {         \
     538             :                                                 msg = createException(MAL, "pyapi3.eval",              \
     539             :                                                                                           SQLSTATE(PY000) "BUNappend failed.\n");          \
     540             :                                                 goto wrapup;                                           \
     541             :                                         }                                                          \
     542             :                                 } else {                                                       \
     543             :                                         /* we try to handle as many types as possible */           \
     544             :                                         pyobject_to_str(                                           \
     545             :                                                 ((PyObject **)&data[(index_offset * ret->count + iu) * \
     546             :                                                                                         ret->memory_size]),                \
     547             :                                                 utf8_size, &utf8_string);                              \
     548             :                                         if (convert_and_append(b, utf8_string, false) != GDK_SUCCEED) {     \
     549             :                                                 msg = createException(MAL, "pyapi3.eval",              \
     550             :                                                                                           SQLSTATE(PY000) "BUNappend failed.\n");          \
     551             :                                                 goto wrapup;                                           \
     552             :                                         }                                                          \
     553             :                                 }                                                              \
     554             :                         }                                                                  \
     555             :                         break;                                                             \
     556             :                 }                                                                      \
     557             :                 default:                                                               \
     558             :                         msg = createException(                                             \
     559             :                                 MAL, "pyapi3.eval",                                            \
     560             :                                 SQLSTATE(PY000) "Unrecognized type. Could not convert to NPY_UNICODE.\n");     \
     561             :                         goto wrapup;                                                       \
     562             :         }                                                                          \
     563             :         b->tnonil = !b->tnil;
     564             : 
     565             : #ifdef HAVE_HGE
     566             : #define NOT_HGE(mtpe) TYPE_##mtpe != TYPE_hge
     567             : #else
     568             : #define NOT_HGE(mtpe) true
     569             : #endif
     570             : 
     571             : #define NP_CREATE_EMPTY_BAT(bat, mtpe)                                     \
     572             :         {                                                                      \
     573             :                 bat = COLnew(seqbase, TYPE_##mtpe, (BUN)ret->count, TRANSIENT);    \
     574             :                 if (bat == NULL) {                                                 \
     575             :                         msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Cannot create column"); \
     576             :                         goto wrapup;                                                   \
     577             :                 }                                                                  \
     578             :                 bat->tkey = false;                                                 \
     579             :                 bat->tsorted = false;                                              \
     580             :                 bat->trevsorted = false;                                           \
     581             :                 bat->tnil = false;                                                 \
     582             :                 bat->tnonil = true;                                                \
     583             :                 BATsetcount(bat, (BUN)ret->count);                                 \
     584             :                 BATsettrivprop(bat);                                               \
     585             :         }
     586             : 
     587             : // This very big #define combines all the previous #defines for one big #define
     588             : // that is responsible for converting a Numpy array (described in the PyReturn
     589             : // object 'ret')
     590             : // to a BAT of type 'mtpe'. This should only be used for numeric BATs (but can
     591             : // be used for any Numpy Array). The resulting BAT will be stored in 'bat'.
     592             : #define NP_CREATE_BAT(bat, mtpe)                                               \
     593             :         {                                                                          \
     594             :                 bool *mask = NULL;                                                     \
     595             :                 char *data = NULL;                                                     \
     596             :                 if (ret->mask_data != NULL) {                                          \
     597             :                         mask = (bool *)ret->mask_data;                                     \
     598             :                 }                                                                      \
     599             :                 if (ret->array_data == NULL) {                                         \
     600             :                         msg =                                                              \
     601             :                                 createException(MAL, "pyapi3.eval",                            \
     602             :                                                                 SQLSTATE(PY000) "No return value stored in the structure.\n"); \
     603             :                         goto wrapup;                                                       \
     604             :                 }                                                                      \
     605             :                 data = (char *)ret->array_data;                                        \
     606             :                 if (!copy && ret->count > 0 &&                                         \
     607             :                         TYPE_##mtpe == PyType_ToBat(ret->result_type) &&                   \
     608             :                         (ret->count * ret->memory_size < BUN_MAX) &&                       \
     609             :                         (ret->numpy_array == NULL ||                                       \
     610             :                          PyArray_FLAGS((PyArrayObject *)ret->numpy_array) &                \
     611             :                                  NPY_ARRAY_OWNDATA)) {                                         \
     612             :                         /*We can only create a direct map if the numpy array type and      \
     613             :                          * target BAT type*/                                               \
     614             :                         /*are identical, otherwise we have to do a conversion.*/           \
     615             :                         if (ret->numpy_array == NULL) {                                    \
     616             :                                 CREATE_BAT_ZEROCOPY(bat, mtpe, STORE_MMAPABS);                 \
     617             :                                 ret->array_data = NULL;                                        \
     618             :                         } else {                                                           \
     619             :                                 CREATE_BAT_ZEROCOPY(bat, mtpe, STORE_CMEM);                    \
     620             :                         }                                                                  \
     621             :                 } else {                                                               \
     622             :                         bat = COLnew(seqbase, TYPE_##mtpe, (BUN)ret->count, TRANSIENT);    \
     623             :                         if (bat == NULL) {                                                 \
     624             :                                 msg = createException(MAL, "pyapi3.eval", SQLSTATE(PY000) "Cannot create column"); \
     625             :                                 goto wrapup;                                                   \
     626             :                         }                                                                  \
     627             :                         bat->tkey = false;                                                 \
     628             :                         bat->tsorted = false;                                              \
     629             :                         bat->trevsorted = false;                                           \
     630             :                         NP_INSERT_BAT(bat, mtpe, 0);                                       \
     631             :                         if (!mask) {                                                       \
     632             :                                 bat->tnil = false;                                             \
     633             :                                 bat->tnonil = false;                                           \
     634             :                         }                                                                  \
     635             :                         BATsetcount(bat, (BUN)ret->count);                                 \
     636             :                         BATsettrivprop(bat);                                               \
     637             :                 }                                                                      \
     638             :         }

Generated by: LCOV version 1.14