LCOV - code coverage report
Current view: top level - sql/backends/monet5/UDF/pyapi3 - emit3.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 119 241 49.4 %
Date: 2021-10-13 02:24:04 Functions: 3 3 100.0 %

          Line data    Source code
       1             : /*
       2             :  * This Source Code Form is subject to the terms of the Mozilla Public
       3             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       4             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       5             :  *
       6             :  * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V.
       7             :  */
       8             : 
       9             : #include "monetdb_config.h"
      10             : #include "emit.h"
      11             : #include "conversion.h"
      12             : #include "convert_loops.h"
      13             : #include "type_conversion.h"
      14             : #include "gdk_interprocess.h"
      15             : #include "gdk_time.h"
      16             : 
      17             : #include "unicode.h"
      18             : 
      19             : #define scalar_convert(tpe)                                                    \
      20             :         {                                                                          \
      21             :                 tpe val = tpe##_nil;                                                   \
      22             :                 msg = pyobject_to_##tpe(&dictEntry, 42, &val);                         \
      23             :                 if (msg != MAL_SUCCEED ||                                              \
      24             :                         BUNappend(self->cols[i].b, &val, false) != GDK_SUCCEED) {          \
      25             :                         if (msg == MAL_SUCCEED)                                            \
      26             :                                 msg = createException(MAL, "pyapi3.emit", SQLSTATE(HY013) "BUNappend failed."); \
      27             :                         goto wrapup;                                                       \
      28             :                 }                                                                      \
      29             :         }
      30             : 
      31          76 : PyObject *PyEmit_Emit(PyEmitObject *self, PyObject *args)
      32             : {
      33             :         size_t i, ai; // iterators
      34             :         ssize_t el_count =
      35             :                 -1; // the amount of elements this emit call will write to the table
      36             :         size_t dict_elements, matched_elements;
      37             :         str msg = MAL_SUCCEED; // return message
      38             :         bool error = false;
      39             : 
      40          76 :         if (!PyDict_Check(args)) {
      41           0 :                 PyErr_SetString(PyExc_TypeError, "need dict");
      42           0 :                 return NULL;
      43             :         }
      44             : 
      45             :         matched_elements = 0;
      46          76 :         dict_elements = PyDict_Size(args);
      47          76 :         if (dict_elements == 0) {
      48           0 :                 PyErr_SetString(PyExc_TypeError,
      49             :                                                 "dict must contain at least one element");
      50           0 :                 return NULL;
      51             :         }
      52             :         {
      53          76 :                 PyObject *items = PyDict_Items(args);
      54         300 :                 for (i = 0; i < dict_elements; i++) {
      55         224 :                         PyObject *tuple = PyList_GetItem(items, i);
      56         224 :                         PyObject *key = PyTuple_GetItem(tuple, 0);
      57         224 :                         PyObject *dictEntry = PyTuple_GetItem(tuple, 1);
      58             :                         ssize_t this_size = 1;
      59         224 :                         this_size = PyType_Size(dictEntry);
      60         224 :                         if (this_size < 0) {
      61           0 :                                 PyErr_Format(
      62             :                                         PyExc_TypeError, "Unsupported Python Object %s",
      63             :                                         PyString_AsString(PyObject_Str(PyObject_Type(dictEntry))));
      64             :                                 Py_DECREF(items);
      65           0 :                                 return NULL;
      66             :                         }
      67         224 :                         if (el_count < 0) {
      68             :                                 el_count = this_size;
      69         148 :                         } else if (el_count != this_size) {
      70             :                                 /* don't use "%zu" since format given to Python */
      71           0 :                                 PyErr_Format(
      72             :                                         PyExc_TypeError, "Element %s has size %zd, but expected an "
      73             :                                                                          "element with size %zd",
      74             :                                         PyString_AsString(PyObject_Str(key)), this_size, el_count);
      75             :                                 Py_DECREF(items);
      76           0 :                                 return NULL;
      77             :                         }
      78             :                 }
      79             :                 Py_DECREF(items);
      80             :         }
      81          76 :         if (el_count == 0) {
      82           0 :                 PyErr_SetString(PyExc_TypeError, "Empty input values supplied");
      83           0 :                 return NULL;
      84             :         }
      85             : 
      86          76 :         if (!self->create_table) {
      87         324 :                 for (i = 0; i < self->ncols; i++) {
      88             :                         PyObject *dictEntry =
      89         249 :                                 PyDict_GetItemString(args, self->cols[i].name);
      90         249 :                         if (dictEntry) {
      91         222 :                                 matched_elements++;
      92             :                         }
      93             :                 }
      94          75 :                 if (matched_elements != dict_elements) {
      95             :                         // not all elements in the dictionary were matched, look for the
      96             :                         // element that was not matched
      97           0 :                         PyObject *keys = PyDict_Keys(args);
      98           0 :                         if (!keys) {
      99           0 :                                 msg = createException(MAL, "pyapi3.emit", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     100           0 :                                 goto wrapup;
     101             :                         }
     102           0 :                         for (i = 0; i < (size_t)PyList_Size(keys); i++) {
     103           0 :                                 PyObject *key = PyList_GetItem(keys, i);
     104           0 :                                 char *val = NULL;
     105             :                                 bool found = false;
     106             : 
     107           0 :                                 msg = pyobject_to_str(&key, 42, &val);
     108           0 :                                 if (msg != MAL_SUCCEED) {
     109             :                                         // one of the keys in the dictionary was not a string
     110           0 :                                         PyErr_Format(
     111             :                                                 PyExc_TypeError,
     112             :                                                 "Could not convert object type %s to a string: %s",
     113             :                                                 PyString_AsString(PyObject_Str(PyObject_Type(key))),
     114             :                                                 msg);
     115           0 :                                         free(val);
     116           0 :                                         goto loop_end;
     117             :                                 }
     118           0 :                                 for (ai = 0; ai < self->ncols; ai++) {
     119           0 :                                         if (strcmp(val, self->cols[ai].name) == 0) {
     120             :                                                 found = true;
     121             :                                                 break;
     122             :                                         }
     123             :                                 }
     124           0 :                                 if (!found) {
     125             :                                         // the current element was present in the dictionary, but it
     126             :                                         // has no matching column
     127           0 :                                         PyErr_Format(PyExc_TypeError,
     128             :                                                                  "Unmatched element \"%s\" in dict", val);
     129             :                                         error = true;
     130           0 :                                         free(val);
     131           0 :                                         goto loop_end;
     132             :                                 }
     133           0 :                                 free(val);
     134             :                         }
     135           0 :                 loop_end:
     136             :                         Py_DECREF(keys);
     137           0 :                         goto wrapup;
     138             :                 }
     139             :         } else {
     140           1 :                 size_t potential_size = self->ncols + PyDict_Size(args);
     141             :                 PyObject *keys;
     142           1 :                 if (potential_size > self->maxcols) {
     143             :                         // allocate space for new columns (if any new columns show up)
     144           1 :                         sql_emit_col *old = self->cols;
     145           1 :                         self->cols = GDKzalloc(sizeof(sql_emit_col) * potential_size);
     146           1 :                         if (self->cols == NULL) {
     147           0 :                                 PyErr_Format(PyExc_TypeError, "Out of memory error");
     148             :                                 error = true;
     149           0 :                                 goto wrapup;
     150             :                         }
     151           1 :                         if (old) {
     152           0 :                                 memcpy(self->cols, old, sizeof(sql_emit_col) * self->maxcols);
     153           0 :                                 GDKfree(old);
     154             :                         }
     155           1 :                         self->maxcols = potential_size;
     156             :                 }
     157           1 :                 keys = PyDict_Keys(args);
     158             :                 // create new columns based on the entries in the dictionary
     159           3 :                 for (i = 0; i < (size_t)PyList_Size(keys); i++) {
     160           2 :                         PyObject *key = PyList_GetItem(keys, i);
     161           2 :                         char *val = NULL;
     162             :                         bool found = false;
     163             : 
     164           2 :                         msg = pyobject_to_str(&key, 42, &val);
     165           2 :                         if (msg != MAL_SUCCEED) {
     166             :                                 // one of the keys in the dictionary was not a string
     167           0 :                                 PyErr_Format(
     168             :                                         PyExc_TypeError,
     169             :                                         "Could not convert object type %s to a string: %s",
     170             :                                         PyString_AsString(PyObject_Str(PyObject_Type(key))), msg);
     171             :                                 error = true;
     172             :                                 Py_DECREF(keys);
     173           0 :                                 free(val);
     174           0 :                                 goto wrapup;
     175             :                         }
     176           3 :                         for (ai = 0; ai < self->ncols; ai++) {
     177           1 :                                 if (strcmp(val, self->cols[ai].name) == 0) {
     178             :                                         found = true;
     179             :                                         break;
     180             :                                 }
     181             :                         }
     182           2 :                         if (!found) {
     183             :                                 // unrecognized column, create the column in the table
     184             :                                 // first infer the type from the value
     185             :                                 // we use NumPy for this by creating an array from the object
     186             :                                 // without specifying the type
     187           2 :                                 PyObject *value = PyDict_GetItem(args, key);
     188           2 :                                 PyObject *array = PyArray_FromAny(
     189             :                                         value, NULL, 0, 0, NPY_ARRAY_CARRAY | NPY_ARRAY_FORCECAST,
     190             :                                         NULL);
     191             :                                 PyArray_Descr *array_type = NULL;
     192             :                                 int bat_type = TYPE_int;
     193           2 :                                 if (!array) {
     194           0 :                                         PyErr_Format(PyExc_TypeError,
     195             :                                                                  "Failed to create NumPy array.");
     196             :                                         error = true;
     197           0 :                                         free(val);
     198           0 :                                         goto wrapup;
     199             :                                 }
     200             :                                 array_type =
     201             :                                         (PyArray_Descr *)PyArray_DESCR((PyArrayObject *)array);
     202           2 :                                 bat_type = PyType_ToBat(array_type->type_num);
     203             :                                 Py_DECREF(array);
     204             : 
     205           2 :                                 if (!(self->cols[self->ncols].b = COLnew(0, bat_type, 0, TRANSIENT))) {
     206           0 :                                         msg = createException(MAL, "pyapi3.emit", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     207           0 :                                         free(val);
     208           0 :                                         goto wrapup;
     209             :                                 }
     210           2 :                                 if (!(self->cols[self->ncols].name = GDKstrdup(val))) {
     211           0 :                                         msg = createException(MAL, "pyapi3.emit", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     212           0 :                                         free(val);
     213           0 :                                         goto wrapup;
     214             :                                 }
     215           2 :                                 self->cols[self->ncols].def = NULL;
     216           2 :                                 if (self->nvals > 0) {
     217             :                                         // insert NULL values up until the current entry
     218           0 :                                         for (ai = 0; ai < self->nvals; ai++) {
     219           0 :                                                 if (BUNappend(self->cols[self->ncols].b,
     220           0 :                                                                           ATOMnilptr(self->cols[self->ncols].b->ttype),
     221             :                                                                           false) != GDK_SUCCEED) {
     222           0 :                                                         msg = createException(MAL, "pyapi3.emit", SQLSTATE(HY013) "BUNappend failed.");
     223           0 :                                                         free(val);
     224           0 :                                                         goto wrapup;
     225             :                                                 }
     226             :                                         }
     227           0 :                                         self->cols[self->ncols].b->tnil = true;
     228           0 :                                         self->cols[self->ncols].b->tnonil = false;
     229           0 :                                         BATsetcount(self->cols[self->ncols].b, self->nvals);
     230             :                                 }
     231           2 :                                 self->ncols++;
     232             :                         }
     233           2 :                         free(val);
     234             :                 }
     235             :         }
     236             : 
     237         273 :         for (i = 0; i < self->ncols; i++) {
     238         209 :                 PyObject *dictEntry = PyDict_GetItemString(args, self->cols[i].name);
     239         209 :                 if (dictEntry && dictEntry != Py_None) {
     240         182 :                         if (PyType_IsPyScalar(dictEntry)) {
     241         141 :                                 if (self->cols[i].b->ttype == TYPE_blob) {
     242             :                                         blob s;
     243           0 :                                         blob* val = &s;
     244           0 :                                         val->nitems = ~(size_t) 0;
     245           0 :                                         msg = pyobject_to_blob(&dictEntry, 42, &val);
     246           0 :                                         if (msg != MAL_SUCCEED ||
     247           0 :                                                 BUNappend(self->cols[i].b, val, false) != GDK_SUCCEED) {
     248           0 :                                                 if (msg == MAL_SUCCEED)
     249           0 :                                                         msg = createException(MAL, "pyapi3.emit", SQLSTATE(HY013) "BUNappend failed.");
     250           0 :                                                 goto wrapup;
     251             :                                         }
     252           0 :                                 GDKfree(val);
     253             :                                 } else {
     254         141 :                                         switch (self->cols[i].b->ttype) {
     255           0 :                                                 case TYPE_bit:
     256           0 :                                                         scalar_convert(bit);
     257           0 :                                                         break;
     258           0 :                                                 case TYPE_bte:
     259           0 :                                                         scalar_convert(bte);
     260           0 :                                                         break;
     261           0 :                                                 case TYPE_sht:
     262           0 :                                                         scalar_convert(sht);
     263           0 :                                                         break;
     264         132 :                                                 case TYPE_int:
     265         132 :                                                         scalar_convert(int);
     266         132 :                                                         break;
     267           0 :                                                 case TYPE_oid:
     268           0 :                                                         scalar_convert(oid);
     269           0 :                                                         break;
     270           0 :                                                 case TYPE_lng:
     271           0 :                                                         scalar_convert(lng);
     272           0 :                                                         break;
     273           0 :                                                 case TYPE_flt:
     274           0 :                                                         scalar_convert(flt);
     275           0 :                                                         break;
     276           4 :                                                 case TYPE_dbl:
     277           4 :                                                         scalar_convert(dbl);
     278           4 :                                                         break;
     279             : #ifdef HAVE_HGE
     280           0 :                                                 case TYPE_hge:
     281           0 :                                                         scalar_convert(hge);
     282           0 :                                                         break;
     283             : #endif
     284           5 :                                                 default: {
     285           5 :                                                         str val = NULL;
     286             :                                                         gdk_return retval;
     287           5 :                                                         msg = pyobject_to_str(&dictEntry, 42, &val);
     288           5 :                                                         if (msg != MAL_SUCCEED) {
     289           0 :                                                                 free(val);
     290           0 :                                                                 goto wrapup;
     291             :                                                         }
     292           5 :                                                         assert(val);
     293           5 :                                                         retval = convert_and_append(self->cols[i].b, val, 0);
     294           5 :                                                         free(val);
     295           5 :                                                         if (retval != GDK_SUCCEED) {
     296           0 :                                                                 msg = createException(MAL, "pyapi3.emit", SQLSTATE(HY013) "BUNappend failed.");
     297           0 :                                                                 goto wrapup;
     298             :                                                         }
     299           5 :                                                 } break;
     300             :                                         }
     301             :                                 }
     302             :                         } else {
     303             :                                 bool *mask = NULL;
     304             :                                 char *data = NULL;
     305             :                                 PyReturn return_struct;
     306             :                                 PyReturn *ret = &return_struct;
     307             :                                 size_t index_offset = 0;
     308             :                                 size_t iu = 0;
     309          41 :                                 if (BATextend(self->cols[i].b, self->nvals + el_count) !=
     310             :                                         GDK_SUCCEED) {
     311           0 :                                         msg = createException(MAL, "pyapi3.emit", SQLSTATE(HY013) "Failed to allocate memory to extend BAT.");
     312           0 :                                         goto wrapup;
     313             :                                 }
     314          41 :                                 msg = PyObject_GetReturnValues(dictEntry, ret);
     315          41 :                                 if (msg != MAL_SUCCEED) {
     316           0 :                                         goto wrapup;
     317             :                                 }
     318          41 :                                 if (ret->array_data == NULL) {
     319           0 :                                         msg = createException(MAL, "pyapi3.emit", SQLSTATE(HY013) "No return value stored in the structure.");
     320           0 :                                         goto wrapup;
     321             :                                 }
     322          41 :                                 mask = (bool *)ret->mask_data;
     323             :                                 data = (char *)ret->array_data;
     324          41 :                                 assert((size_t)el_count == (size_t)ret->count);
     325             : 
     326             :                                 /* we're not maintaining properties */
     327          41 :                                 self->cols[i].b->tsorted = false;
     328          41 :                                 self->cols[i].b->trevsorted = false;
     329             : 
     330          41 :                                 switch (self->cols[i].b->ttype) {
     331           0 :                                         case TYPE_bit:
     332           0 :                                                 NP_INSERT_BAT(self->cols[i].b, bit, self->nvals);
     333           0 :                                                 break;
     334           0 :                                         case TYPE_bte:
     335           0 :                                                 NP_INSERT_BAT(self->cols[i].b, bte, self->nvals);
     336           0 :                                                 break;
     337           0 :                                         case TYPE_sht:
     338           0 :                                                 NP_INSERT_BAT(self->cols[i].b, sht, self->nvals);
     339           0 :                                                 break;
     340          28 :                                         case TYPE_int:
     341          84 :                                                 NP_INSERT_BAT(self->cols[i].b, int, self->nvals);
     342          28 :                                                 break;
     343           0 :                                         case TYPE_oid:
     344           0 :                                                 NP_INSERT_BAT(self->cols[i].b, oid, self->nvals);
     345           0 :                                                 break;
     346           0 :                                         case TYPE_lng:
     347           0 :                                                 NP_INSERT_BAT(self->cols[i].b, lng, self->nvals);
     348           0 :                                                 break;
     349           0 :                                         case TYPE_flt:
     350           0 :                                                 NP_INSERT_BAT(self->cols[i].b, flt, self->nvals);
     351           0 :                                                 break;
     352           0 :                                         case TYPE_dbl:
     353           0 :                                                 NP_INSERT_BAT(self->cols[i].b, dbl, self->nvals);
     354           0 :                                                 break;
     355             : #ifdef HAVE_HGE
     356           0 :                                         case TYPE_hge:
     357           0 :                                                 NP_INSERT_BAT(self->cols[i].b, hge, self->nvals);
     358           0 :                                                 break;
     359             : #endif
     360          13 :                                         default: {
     361          13 :                                                 char *utf8_string = NULL;
     362          13 :                                                 if (ret->result_type != NPY_OBJECT) {
     363          13 :                                                         utf8_string = GDKzalloc(utf8string_minlength +
     364             :                                                                                                         ret->memory_size + 1);
     365          13 :                                                         utf8_string[utf8string_minlength +
     366          13 :                                                                                 ret->memory_size] = '\0';
     367             :                                                 }
     368          48 :                                                 NP_INSERT_STRING_BAT(self->cols[i].b);
     369          13 :                                                 GDKfree(utf8_string);
     370             :                                         }
     371             :                                 }
     372          41 :                                 self->cols[i].b->tnonil = !self->cols[i].b->tnil;
     373          41 :                                 if (ret->numpy_array) {
     374             :                                         Py_DECREF(ret->numpy_array);
     375             :                                 }
     376          41 :                                 if (ret->numpy_mask) {
     377             :                                         Py_DECREF(ret->numpy_mask);
     378             :                                 }
     379             :                         }
     380             :                 } else {
     381          27 :                         if (self->cols[i].def != NULL) {
     382          12 :                                 msg = createException(MAL, "pyapi3.emit", "Inserting into columns with default values is not supported currently.");
     383          12 :                                 goto wrapup;
     384             :                         }
     385          31 :                         for (ai = 0; ai < (size_t)el_count; ai++) {
     386          16 :                                 if (BUNappend(self->cols[i].b,
     387          16 :                                                           ATOMnilptr(self->cols[i].b->ttype),
     388             :                                                           false) != GDK_SUCCEED) {
     389           0 :                                         goto wrapup;
     390             :                                 }
     391             :                         }
     392          15 :                         self->cols[i].b->tnil = true;
     393          15 :                         self->cols[i].b->tnonil = false;
     394             :                 }
     395         197 :                 BATsetcount(self->cols[i].b, self->nvals + el_count);
     396             :         }
     397             : 
     398          64 :         self->nvals += el_count;
     399          76 : wrapup:
     400          76 :         if (msg != MAL_SUCCEED) {
     401          12 :                 PyErr_Format(PyExc_TypeError, "Failed conversion: %s", msg);
     402          12 :                 freeException(msg);
     403          64 :         } else if (!error) {
     404          64 :                 Py_RETURN_NONE;
     405             :         }
     406             :         return NULL;
     407             : }
     408             : 
     409             : static PyMethodDef _emitObject_methods[] = {
     410             :         {"emit", (PyCFunction)PyEmit_Emit, METH_O,
     411             :          "emit(dictionary) -> returns parsed values for table insertion"},
     412             :         {NULL, NULL, 0, NULL} /* Sentinel */
     413             : };
     414             : 
     415             : PyTypeObject PyEmitType = {
     416             :         .ob_base.ob_base.ob_refcnt = 1,
     417             :         .tp_name = "monetdb._emit",
     418             :         .tp_basicsize = sizeof(PyEmitObject),
     419             :         .tp_hash = (hashfunc)PyObject_HashNotImplemented,
     420             :         .tp_flags = Py_TPFLAGS_DEFAULT,
     421             :         .tp_doc = "Value Emitter",
     422             :         .tp_methods = _emitObject_methods,
     423             :         .tp_alloc = PyType_GenericAlloc,
     424             :         .tp_new = PyType_GenericNew,
     425             :         .tp_free = PyObject_Del,
     426             : };
     427             : 
     428          25 : PyObject *PyEmit_Create(sql_emit_col *cols, size_t ncols)
     429             : {
     430             :         register PyEmitObject *op;
     431             : 
     432          25 :         op = (PyEmitObject *)PyObject_MALLOC(sizeof(PyEmitObject));
     433          25 :         if (op == NULL)
     434           0 :                 return PyErr_NoMemory();
     435          25 :         PyObject_Init((PyObject *)op, &PyEmitType);
     436          25 :         op->cols = cols;
     437          25 :         op->ncols = ncols;
     438          25 :         op->maxcols = ncols;
     439          25 :         op->nvals = 0;
     440          25 :         op->create_table = cols == NULL;
     441          25 :         return (PyObject *)op;
     442             : }
     443             : 
     444           8 : str _emit_init(void)
     445             : {
     446           8 :         _import_array();
     447           8 :         if (PyType_Ready(&PyEmitType) < 0)
     448           0 :                 return createException(MAL, "pyapi3.eval",
     449             :                                                            SQLSTATE(PY000) "Failed to initialize emit type.");
     450             :         return MAL_SUCCEED;
     451             : }

Generated by: LCOV version 1.14