LCOV - code coverage report
Current view: top level - sql/backends/monet5/UDF/pyapi3 - emit3.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 144 263 54.8 %
Date: 2020-06-29 20:00:14 Functions: 3 3 100.0 %

          Line data    Source code
       1             : /*
       2             :  * This Source Code Form is subject to the terms of the Mozilla Public
       3             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       4             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       5             :  *
       6             :  * Copyright 1997 - July 2008 CWI, August 2008 - 2020 MonetDB B.V.
       7             :  */
       8             : 
       9             : #include "monetdb_config.h"
      10             : #include "emit.h"
      11             : #include "conversion.h"
      12             : #include "convert_loops.h"
      13             : #include "type_conversion.h"
      14             : #include "gdk_interprocess.h"
      15             : #include "gdk_time.h"
      16             : 
      17             : #include "unicode.h"
      18             : 
      19             : #define scalar_convert(tpe)                                                    \
      20             :         {                                                                          \
      21             :                 tpe val = tpe##_nil;                                                   \
      22             :                 msg = pyobject_to_##tpe(&dictEntry, 42, &val);                         \
      23             :                 if (msg != MAL_SUCCEED ||                                              \
      24             :                         BUNappend(self->cols[i].b, &val, false) != GDK_SUCCEED) {              \
      25             :                         if (msg == MAL_SUCCEED)                                            \
      26             :                                 msg = GDKstrdup("BUNappend failed.");                          \
      27             :                         goto wrapup;                                                       \
      28             :                 }                                                                      \
      29             :         }
      30             : 
      31          76 : PyObject *PyEmit_Emit(PyEmitObject *self, PyObject *args)
      32             : {
      33          76 :         size_t i, ai; // iterators
      34          76 :         ssize_t el_count =
      35             :                 -1; // the amount of elements this emit call will write to the table
      36          76 :         size_t dict_elements, matched_elements;
      37          76 :         str msg = MAL_SUCCEED; // return message
      38          76 :         bool error = false;
      39             : 
      40          76 :         if (!PyDict_Check(args)) {
      41           0 :                 PyErr_SetString(PyExc_TypeError, "need dict");
      42           0 :                 return NULL;
      43             :         }
      44             : 
      45          76 :         matched_elements = 0;
      46          76 :         dict_elements = PyDict_Size(args);
      47          76 :         if (dict_elements == 0) {
      48           0 :                 PyErr_SetString(PyExc_TypeError,
      49             :                                                 "dict must contain at least one element");
      50           0 :                 return NULL;
      51             :         }
      52             :         {
      53          76 :                 PyObject *items = PyDict_Items(args);
      54         376 :                 for (i = 0; i < dict_elements; i++) {
      55         224 :                         PyObject *tuple = PyList_GetItem(items, i);
      56         224 :                         PyObject *key = PyTuple_GetItem(tuple, 0);
      57         224 :                         PyObject *dictEntry = PyTuple_GetItem(tuple, 1);
      58         224 :                         ssize_t this_size = 1;
      59         224 :                         this_size = PyType_Size(dictEntry);
      60         224 :                         if (this_size < 0) {
      61           0 :                                 PyErr_Format(
      62             :                                         PyExc_TypeError, "Unsupported Python Object %s",
      63             :                                         PyString_AsString(PyObject_Str(PyObject_Type(dictEntry))));
      64           0 :                                 Py_DECREF(items);
      65           0 :                                 return NULL;
      66             :                         }
      67         224 :                         if (el_count < 0) {
      68             :                                 el_count = this_size;
      69         148 :                         } else if (el_count != this_size) {
      70             :                                 /* don't use "%zu" since format given to Python */
      71           0 :                                 PyErr_Format(
      72             :                                         PyExc_TypeError, "Element %s has size %zd, but expected an "
      73             :                                                                          "element with size %zd",
      74             :                                         PyString_AsString(PyObject_Str(key)), this_size, el_count);
      75           0 :                                 Py_DECREF(items);
      76           0 :                                 return NULL;
      77             :                         }
      78             :                 }
      79          76 :                 Py_DECREF(items);
      80             :         }
      81          76 :         if (el_count == 0) {
      82           0 :                 PyErr_SetString(PyExc_TypeError, "Empty input values supplied");
      83           0 :                 return NULL;
      84             :         }
      85             : 
      86          76 :         if (!self->create_table) {
      87         324 :                 for (i = 0; i < self->ncols; i++) {
      88         249 :                         PyObject *dictEntry =
      89         249 :                                 PyDict_GetItemString(args, self->cols[i].name);
      90         249 :                         if (dictEntry) {
      91         222 :                                 matched_elements++;
      92             :                         }
      93             :                 }
      94          75 :                 if (matched_elements != dict_elements) {
      95             :                         // not all elements in the dictionary were matched, look for the
      96             :                         // element that was not matched
      97           0 :                         PyObject *keys = PyDict_Keys(args);
      98           0 :                         if (!keys) {
      99           0 :                                 msg = GDKstrdup(MAL_MALLOC_FAIL);
     100           0 :                                 goto wrapup;
     101             :                         }
     102           0 :                         for (i = 0; i < (size_t)PyList_Size(keys); i++) {
     103           0 :                                 PyObject *key = PyList_GetItem(keys, i);
     104           0 :                                 char *val = NULL;
     105           0 :                                 bool found = false;
     106             : 
     107           0 :                                 msg = pyobject_to_str(&key, 42, &val);
     108           0 :                                 if (msg != MAL_SUCCEED) {
     109             :                                         // one of the keys in the dictionary was not a string
     110           0 :                                         PyErr_Format(
     111             :                                                 PyExc_TypeError,
     112             :                                                 "Could not convert object type %s to a string: %s",
     113             :                                                 PyString_AsString(PyObject_Str(PyObject_Type(key))),
     114             :                                                 msg);
     115           0 :                                         goto loop_end;
     116             :                                 }
     117           0 :                                 for (ai = 0; ai < self->ncols; ai++) {
     118           0 :                                         if (strcmp(val, self->cols[ai].name) == 0) {
     119             :                                                 found = true;
     120             :                                                 break;
     121             :                                         }
     122             :                                 }
     123           0 :                                 if (!found) {
     124             :                                         // the current element was present in the dictionary, but it
     125             :                                         // has no matching column
     126           0 :                                         PyErr_Format(PyExc_TypeError,
     127             :                                                                  "Unmatched element \"%s\" in dict", val);
     128           0 :                                         error = true;
     129           0 :                                         goto loop_end;
     130             :                                 }
     131             :                         }
     132           0 :                 loop_end:
     133           0 :                         Py_DECREF(keys);
     134           0 :                         goto wrapup;
     135             :                 }
     136             :         } else {
     137           1 :                 size_t potential_size = self->ncols + PyDict_Size(args);
     138           1 :                 PyObject *keys;
     139           1 :                 if (potential_size > self->maxcols) {
     140             :                         // allocate space for new columns (if any new columns show up)
     141           1 :                         sql_emit_col *old = self->cols;
     142           1 :                         self->cols = GDKzalloc(sizeof(sql_emit_col) * potential_size);
     143           1 :                         if (self->cols == NULL) {
     144           0 :                                 PyErr_Format(PyExc_TypeError, "Out of memory error");
     145           0 :                                 error = true;
     146           0 :                                 goto wrapup;
     147             :                         }
     148           1 :                         if (old) {
     149           0 :                                 memcpy(self->cols, old, sizeof(sql_emit_col) * self->maxcols);
     150           0 :                                 GDKfree(old);
     151             :                         }
     152           1 :                         self->maxcols = potential_size;
     153             :                 }
     154           1 :                 keys = PyDict_Keys(args);
     155             :                 // create new columns based on the entries in the dictionary
     156           4 :                 for (i = 0; i < (size_t)PyList_Size(keys); i++) {
     157           2 :                         PyObject *key = PyList_GetItem(keys, i);
     158           2 :                         char *val = NULL;
     159           2 :                         bool found = false;
     160             : 
     161           2 :                         msg = pyobject_to_str(&key, 42, &val);
     162           2 :                         if (msg != MAL_SUCCEED) {
     163             :                                 // one of the keys in the dictionary was not a string
     164           0 :                                 PyErr_Format(
     165             :                                         PyExc_TypeError,
     166             :                                         "Could not convert object type %s to a string: %s",
     167             :                                         PyString_AsString(PyObject_Str(PyObject_Type(key))), msg);
     168           0 :                                 error = true;
     169           0 :                                 Py_DECREF(keys);
     170           0 :                                 goto wrapup;
     171             :                         }
     172           3 :                         for (ai = 0; ai < self->ncols; ai++) {
     173           1 :                                 if (strcmp(val, self->cols[ai].name) == 0) {
     174             :                                         found = true;
     175             :                                         break;
     176             :                                 }
     177             :                         }
     178           2 :                         if (!found) {
     179             :                                 // unrecognized column, create the column in the table
     180             :                                 // first infer the type from the value
     181             :                                 // we use NumPy for this by creating an array from the object
     182             :                                 // without specifying the type
     183           2 :                                 PyObject *value = PyDict_GetItem(args, key);
     184           2 :                                 PyObject *array = PyArray_FromAny(
     185             :                                         value, NULL, 0, 0, NPY_ARRAY_CARRAY | NPY_ARRAY_FORCECAST,
     186             :                                         NULL);
     187           2 :                                 PyArray_Descr *array_type = NULL;
     188           2 :                                 int bat_type = TYPE_int;
     189           2 :                                 if (!array) {
     190           0 :                                         PyErr_Format(PyExc_TypeError,
     191             :                                                                  "Failed to create NumPy array.");
     192           0 :                                         error = true;
     193           0 :                                         goto wrapup;
     194             :                                 }
     195           2 :                                 array_type =
     196           2 :                                         (PyArray_Descr *)PyArray_DESCR((PyArrayObject *)array);
     197           2 :                                 bat_type = PyType_ToBat(array_type->type_num);
     198           2 :                                 Py_DECREF(array);
     199             : 
     200           2 :                                 self->cols[self->ncols].b = COLnew(0, bat_type, 0, TRANSIENT);
     201           2 :                                 self->cols[self->ncols].name = GDKstrdup(val);
     202           2 :                                 self->cols[self->ncols].def = NULL;
     203           2 :                                 if (self->nvals > 0) {
     204             :                                         // insert NULL values up until the current entry
     205           0 :                                         for (ai = 0; ai < self->nvals; ai++) {
     206           0 :                                                 if (BUNappend(self->cols[self->ncols].b,
     207           0 :                                                                           ATOMnil(self->cols[self->ncols].b->ttype),
     208             :                                                                           false) != GDK_SUCCEED) {
     209           0 :                                                         msg = GDKstrdup("BUNappend failed.");
     210           0 :                                                         goto wrapup;
     211             :                                                 }
     212             :                                         }
     213           0 :                                         self->cols[self->ncols].b->tnil = true;
     214           0 :                                         self->cols[self->ncols].b->tnonil = false;
     215           0 :                                         BATsetcount(self->cols[self->ncols].b, self->nvals);
     216             :                                 }
     217           2 :                                 self->ncols++;
     218             :                         }
     219             :                 }
     220             :         }
     221             : 
     222         273 :         for (i = 0; i < self->ncols; i++) {
     223         209 :                 PyObject *dictEntry = PyDict_GetItemString(args, self->cols[i].name);
     224         209 :                 if (dictEntry && dictEntry != Py_None) {
     225         182 :                         if (PyType_IsPyScalar(dictEntry)) {
     226         141 :                                 if (self->cols[i].b->ttype == TYPE_blob) {
     227           0 :                                         blob s;
     228           0 :                                         blob* val = &s;
     229           0 :                                         val->nitems = ~(size_t) 0;
     230           0 :                                         msg = pyobject_to_blob(&dictEntry, 42, &val);
     231           0 :                                         if (msg != MAL_SUCCEED ||
     232           0 :                                                 BUNappend(self->cols[i].b, val, false) != GDK_SUCCEED) {
     233           0 :                                                 if (msg == MAL_SUCCEED)
     234           0 :                                                         msg = GDKstrdup("BUNappend failed.");
     235           0 :                                                 goto wrapup;
     236             :                                         }
     237           0 :                                 GDKfree(val);
     238             :                                 } else {
     239         141 :                                         switch (self->cols[i].b->ttype) {
     240           0 :                                                 case TYPE_bit:
     241           0 :                                                         scalar_convert(bit);
     242           0 :                                                         break;
     243           0 :                                                 case TYPE_bte:
     244           0 :                                                         scalar_convert(bte);
     245           0 :                                                         break;
     246           0 :                                                 case TYPE_sht:
     247           0 :                                                         scalar_convert(sht);
     248           0 :                                                         break;
     249         132 :                                                 case TYPE_int:
     250         132 :                                                         scalar_convert(int);
     251         132 :                                                         break;
     252           0 :                                                 case TYPE_oid:
     253           0 :                                                         scalar_convert(oid);
     254           0 :                                                         break;
     255           0 :                                                 case TYPE_lng:
     256           0 :                                                         scalar_convert(lng);
     257           0 :                                                         break;
     258           0 :                                                 case TYPE_flt:
     259           0 :                                                         scalar_convert(flt);
     260           0 :                                                         break;
     261           4 :                                                 case TYPE_dbl:
     262           4 :                                                         scalar_convert(dbl);
     263           4 :                                                         break;
     264             : #ifdef HAVE_HGE
     265           0 :                                                 case TYPE_hge:
     266           0 :                                                         scalar_convert(hge);
     267           0 :                                                         break;
     268             : #endif
     269           5 :                                                 default: {
     270           5 :                                                         str val = NULL;
     271           5 :                                                         gdk_return retval;
     272           5 :                                                         msg = pyobject_to_str(&dictEntry, 42, &val);
     273           5 :                                                         if (msg != MAL_SUCCEED) {
     274           0 :                                                                 goto wrapup;
     275             :                                                         }
     276           5 :                                                         assert(val);
     277           5 :                                                         retval = convert_and_append(self->cols[i].b, val, 0);
     278           5 :                                                         free(val);
     279           5 :                                                         if (retval != GDK_SUCCEED) {
     280           0 :                                                                 msg = GDKstrdup("BUNappend failed.");
     281           0 :                                                                 goto wrapup;
     282             :                                                         }
     283           5 :                                                 } break;
     284             :                                         }
     285             :                                 }
     286             :                         } else {
     287          41 :                                 bool *mask = NULL;
     288          41 :                                 char *data = NULL;
     289          41 :                                 PyReturn return_struct;
     290          41 :                                 PyReturn *ret = &return_struct;
     291          41 :                                 size_t index_offset = 0;
     292          41 :                                 size_t iu = 0;
     293          41 :                                 if (BATextend(self->cols[i].b, self->nvals + el_count) !=
     294             :                                         GDK_SUCCEED) {
     295           0 :                                         msg = GDKstrdup("Failed to allocate memory to extend BAT.");
     296           0 :                                         goto wrapup;
     297             :                                 }
     298          41 :                                 msg = PyObject_GetReturnValues(dictEntry, ret);
     299          41 :                                 if (msg != MAL_SUCCEED) {
     300           0 :                                         goto wrapup;
     301             :                                 }
     302          41 :                                 if (ret->array_data == NULL) {
     303           0 :                                         msg = GDKstrdup("No return value stored in the structure.");
     304           0 :                                         goto wrapup;
     305             :                                 }
     306          41 :                                 mask = (bool *)ret->mask_data;
     307          41 :                                 data = (char *)ret->array_data;
     308          41 :                                 assert((size_t)el_count == (size_t)ret->count);
     309             : 
     310             :                                 /* we're not maintaining properties */
     311          41 :                                 self->cols[i].b->tsorted = false;
     312          41 :                                 self->cols[i].b->trevsorted = false;
     313             : 
     314          41 :                                 switch (self->cols[i].b->ttype) {
     315           0 :                                         case TYPE_bit:
     316           0 :                                                 NP_INSERT_BAT(self->cols[i].b, bit, self->nvals);
     317           0 :                                                 break;
     318           0 :                                         case TYPE_bte:
     319           0 :                                                 NP_INSERT_BAT(self->cols[i].b, bte, self->nvals);
     320           0 :                                                 break;
     321           0 :                                         case TYPE_sht:
     322           0 :                                                 NP_INSERT_BAT(self->cols[i].b, sht, self->nvals);
     323           0 :                                                 break;
     324          28 :                                         case TYPE_int:
     325          84 :                                                 NP_INSERT_BAT(self->cols[i].b, int, self->nvals);
     326          28 :                                                 break;
     327           0 :                                         case TYPE_oid:
     328           0 :                                                 NP_INSERT_BAT(self->cols[i].b, oid, self->nvals);
     329           0 :                                                 break;
     330           0 :                                         case TYPE_lng:
     331           0 :                                                 NP_INSERT_BAT(self->cols[i].b, lng, self->nvals);
     332           0 :                                                 break;
     333           0 :                                         case TYPE_flt:
     334           0 :                                                 NP_INSERT_BAT(self->cols[i].b, flt, self->nvals);
     335           0 :                                                 break;
     336           0 :                                         case TYPE_dbl:
     337           0 :                                                 NP_INSERT_BAT(self->cols[i].b, dbl, self->nvals);
     338           0 :                                                 break;
     339             : #ifdef HAVE_HGE
     340           0 :                                         case TYPE_hge:
     341           0 :                                                 NP_INSERT_BAT(self->cols[i].b, hge, self->nvals);
     342           0 :                                                 break;
     343             : #endif
     344          13 :                                         default: {
     345          13 :                                                 char *utf8_string = NULL;
     346          13 :                                                 if (ret->result_type != NPY_OBJECT) {
     347          13 :                                                         utf8_string = GDKzalloc(utf8string_minlength +
     348             :                                                                                                         ret->memory_size + 1);
     349          13 :                                                         utf8_string[utf8string_minlength +
     350          13 :                                                                                 ret->memory_size] = '\0';
     351             :                                                 }
     352          48 :                                                 NP_INSERT_STRING_BAT(self->cols[i].b);
     353          13 :                                                 GDKfree(utf8_string);
     354             :                                         }
     355             :                                 }
     356          41 :                                 self->cols[i].b->tnonil = !self->cols[i].b->tnil;
     357          41 :                                 if (ret->numpy_array) {
     358          41 :                                         Py_DECREF(ret->numpy_array);
     359             :                                 }
     360          41 :                                 if (ret->numpy_mask) {
     361          41 :                                         Py_DECREF(ret->numpy_mask);
     362             :                                 }
     363             :                         }
     364             :                 } else {
     365          27 :                         if (self->cols[i].def != NULL) {
     366          12 :                                 msg = GDKstrdup("Inserting into columns with default values is not supported currently.");
     367          12 :                                 goto wrapup;
     368             :                         }
     369          31 :                         for (ai = 0; ai < (size_t)el_count; ai++) {
     370          16 :                                 if (BUNappend(self->cols[i].b,
     371          16 :                                                           ATOMnil(self->cols[i].b->ttype),
     372             :                                                           false) != GDK_SUCCEED) {
     373           0 :                                         goto wrapup;
     374             :                                 }
     375             :                         }
     376          15 :                         self->cols[i].b->tnil = true;
     377          15 :                         self->cols[i].b->tnonil = false;
     378             :                 }
     379         197 :                 BATsetcount(self->cols[i].b, self->nvals + el_count);
     380             :         }
     381             : 
     382          64 :         self->nvals += el_count;
     383          76 : wrapup:
     384          76 :         if (msg != MAL_SUCCEED) {
     385          12 :                 PyErr_Format(PyExc_TypeError, "Failed conversion: %s", msg);
     386          64 :         } else if (!error) {
     387          64 :                 Py_RETURN_NONE;
     388             :         }
     389             :         return NULL;
     390             : }
     391             : 
     392             : static PyMethodDef _emitObject_methods[] = {
     393             :         {"emit", (PyCFunction)PyEmit_Emit, METH_O,
     394             :          "emit(dictionary) -> returns parsed values for table insertion"},
     395             :         {NULL, NULL, 0, NULL} /* Sentinel */
     396             : };
     397             : 
     398             : PyTypeObject PyEmitType = {
     399             :         .ob_base.ob_base.ob_refcnt = 1,
     400             :         .tp_name = "monetdb._emit",
     401             :         .tp_basicsize = sizeof(PyEmitObject),
     402             :         .tp_hash = (hashfunc)PyObject_HashNotImplemented,
     403             :         .tp_flags = Py_TPFLAGS_DEFAULT,
     404             :         .tp_doc = "Value Emitter",
     405             :         .tp_methods = _emitObject_methods,
     406             :         .tp_alloc = PyType_GenericAlloc,
     407             :         .tp_new = PyType_GenericNew,
     408             :         .tp_free = PyObject_Del,
     409             : };
     410             : 
     411          25 : PyObject *PyEmit_Create(sql_emit_col *cols, size_t ncols)
     412             : {
     413          25 :         register PyEmitObject *op;
     414             : 
     415          25 :         op = (PyEmitObject *)PyObject_MALLOC(sizeof(PyEmitObject));
     416          25 :         if (op == NULL)
     417           0 :                 return PyErr_NoMemory();
     418          25 :         PyObject_Init((PyObject *)op, &PyEmitType);
     419          25 :         op->cols = cols;
     420          25 :         op->ncols = ncols;
     421          25 :         op->maxcols = ncols;
     422          25 :         op->nvals = 0;
     423          25 :         op->create_table = cols == NULL;
     424          25 :         return (PyObject *)op;
     425             : }
     426             : 
     427           7 : str _emit_init(void)
     428             : {
     429           7 :         _import_array();
     430           7 :         if (PyType_Ready(&PyEmitType) < 0)
     431           0 :                 return createException(MAL, "pyapi3.eval",
     432             :                                                            SQLSTATE(PY000) "Failed to initialize emit type.");
     433             :         return MAL_SUCCEED;
     434             : }

Generated by: LCOV version 1.14