LCOV - code coverage report
Current view: top level - tools/monetdbe - monetdbe.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 380 1512 25.1 %
Date: 2021-09-14 22:17:06 Functions: 29 69 42.0 %

          Line data    Source code
       1             : /*
       2             :  * This Source Code Form is subject to the terms of the Mozilla Public
       3             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       4             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       5             :  *
       6             :  * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V.
       7             :  */
       8             : 
       9             : #include "monetdb_config.h"
      10             : 
      11             : #include "monetdbe.h"
      12             : #include "gdk.h"
      13             : #include "mal.h"
      14             : #include "mal_client.h"
      15             : #include "mal_embedded.h"
      16             : #include "mal_backend.h"
      17             : #include "mal_builder.h"
      18             : #include "opt_prelude.h"
      19             : #include "blob.h"
      20             : #include "sql_mvc.h"
      21             : #include "sql_catalog.h"
      22             : #include "sql_gencode.h"
      23             : #include "sql_semantic.h"
      24             : #include "sql_scenario.h"
      25             : #include "sql_optimizer.h"
      26             : #include "rel_exp.h"
      27             : #include "rel_rel.h"
      28             : #include "rel_updates.h"
      29             : #include "monet_options.h"
      30             : #include "mapi.h"
      31             : #include "monetdbe_mapi.h"
      32             : #include "remote.h"
      33             : #include "sql.h"
      34             : #include "sql_result.h"
      35             : 
      36             : #define UNUSED(x) (void)(x)
      37             : 
      38             : static int
      39           0 : monetdbe_type(monetdbe_types t) {
      40           0 :         switch(t) {
      41             :         case monetdbe_bool: return TYPE_bit;
      42           0 :         case monetdbe_int8_t: return TYPE_bte;
      43           0 :         case monetdbe_int16_t: return TYPE_sht;
      44           0 :         case monetdbe_int32_t: return TYPE_int;
      45           0 :         case monetdbe_int64_t: return TYPE_lng;
      46             : #ifdef HAVE_HGE
      47           0 :         case monetdbe_int128_t: return TYPE_hge;
      48             : #endif
      49           0 :         case monetdbe_size_t: return TYPE_oid;
      50           0 :         case monetdbe_float: return TYPE_flt;
      51           0 :         case monetdbe_double: return TYPE_dbl;
      52           0 :         case monetdbe_str: return TYPE_str;
      53           0 :         case monetdbe_blob: return TYPE_blob;
      54           0 :         case monetdbe_date: return TYPE_date;
      55           0 :         case monetdbe_time: return TYPE_daytime;
      56           0 :         case monetdbe_timestamp: return TYPE_timestamp;
      57           0 :         default:
      58           0 :                 return -1;
      59             :         }
      60             : }
      61             : 
      62             : static monetdbe_types
      63           0 : embedded_type(int t) {
      64           0 :         switch(t) {
      65             :         case TYPE_bit: return monetdbe_bool;
      66           0 :         case TYPE_bte: return monetdbe_int8_t;
      67           0 :         case TYPE_sht: return monetdbe_int16_t;
      68           0 :         case TYPE_int: return monetdbe_int32_t;
      69           0 :         case TYPE_lng: return monetdbe_int64_t;
      70             : #ifdef HAVE_HGE
      71           0 :         case TYPE_hge: return monetdbe_int128_t;
      72             : #endif
      73           0 :         case TYPE_oid: return monetdbe_size_t;
      74           0 :         case TYPE_flt: return monetdbe_float;
      75           0 :         case TYPE_dbl: return monetdbe_double;
      76           0 :         case TYPE_str: return monetdbe_str;
      77           0 :         case TYPE_date: return monetdbe_date;
      78           0 :         case TYPE_daytime: return monetdbe_time;
      79           0 :         case TYPE_timestamp: return monetdbe_timestamp;
      80           0 :         default:
      81           0 :                 if (t==TYPE_blob)
      82           0 :                         return monetdbe_blob;
      83             :                 return monetdbe_type_unknown;
      84             :         }
      85             : }
      86             : 
      87             : typedef struct {
      88             :         Client c;
      89             :         char *msg;
      90             :         monetdbe_data_blob blob_null;
      91             :         monetdbe_data_date date_null;
      92             :         monetdbe_data_time time_null;
      93             :         monetdbe_data_timestamp timestamp_null;
      94             :         str mid;
      95             : } monetdbe_database_internal;
      96             : 
      97             : typedef struct {
      98             :         monetdbe_result res;
      99             :         int type;
     100             :         res_table *monetdbe_resultset;
     101             :         monetdbe_column **converted_columns;
     102             :         monetdbe_database_internal *mdbe;
     103             : } monetdbe_result_internal;
     104             : 
     105             : typedef struct {
     106             :         monetdbe_statement res;
     107             :         ValRecord *data;
     108             :         ValPtr *args;   /* only used during calls */
     109             :         int retc;
     110             :         monetdbe_database_internal *mdbe;
     111             :         cq *q;
     112             : } monetdbe_stmt_internal;
     113             : 
     114             : static MT_Lock embedded_lock = MT_LOCK_INITIALIZER(embedded_lock);
     115             : static bool monetdbe_embedded_initialized = false;
     116             : static char *monetdbe_embedded_url = NULL;
     117             : static int open_dbs = 0;
     118             : 
     119             : static void data_from_date(date d, monetdbe_data_date *ptr);
     120             : static void data_from_time(daytime d, monetdbe_data_time *ptr);
     121             : static void data_from_timestamp(timestamp d, monetdbe_data_timestamp *ptr);
     122             : static timestamp timestamp_from_data(monetdbe_data_timestamp *ptr);
     123             : static date date_from_data(monetdbe_data_date *ptr);
     124             : static daytime time_from_data(monetdbe_data_time *ptr);
     125             : 
     126             : static char* monetdbe_cleanup_result_internal(monetdbe_database_internal *mdbe, monetdbe_result_internal* res);
     127             : 
     128             : static int
     129           0 : date_is_null(monetdbe_data_date *value)
     130             : {
     131             :         monetdbe_data_date null_value;
     132           0 :         data_from_date(date_nil, &null_value);
     133           0 :         return value->year == null_value.year && value->month == null_value.month &&
     134             :                    value->day == null_value.day;
     135             : }
     136             : 
     137             : static int
     138           0 : time_is_null(monetdbe_data_time *value)
     139             : {
     140             :         monetdbe_data_time null_value;
     141           0 :         data_from_time(daytime_nil, &null_value);
     142           0 :         return value->hours == null_value.hours &&
     143           0 :                    value->minutes == null_value.minutes &&
     144           0 :                    value->seconds == null_value.seconds && value->ms == null_value.ms;
     145             : }
     146             : 
     147             : static int
     148           0 : timestamp_is_null(monetdbe_data_timestamp *value)
     149             : {
     150           0 :         return is_timestamp_nil(timestamp_from_data(value));
     151             : }
     152             : 
     153             : static int
     154           2 : str_is_null(char **value)
     155             : {
     156           2 :         return !value || *value == NULL;
     157             : }
     158             : 
     159             : static int
     160           0 : blob_is_null(monetdbe_data_blob *value)
     161             : {
     162           0 :         return !value || value->data == NULL;
     163             : }
     164             : 
     165             : const char *
     166           0 : monetdbe_version(void)
     167             : {
     168           0 :         return MONETDBE_VERSION;
     169             : }
     170             : 
     171             : static void
     172             : clear_error( monetdbe_database_internal *mdbe)
     173             : {
     174           6 :         if (mdbe->msg)
     175           0 :                 freeException(mdbe->msg);
     176           6 :         mdbe->msg = NULL;
     177           0 : }
     178             : 
     179             : static char*
     180           0 : set_error( monetdbe_database_internal *mdbe, char *err)
     181             : {
     182           0 :         if (!err)
     183             :                 return err;
     184           0 :         if (mdbe->msg) /* keep first error */
     185           0 :                 freeException(err);
     186             :         else
     187           0 :                 mdbe->msg = err;
     188           0 :         return mdbe->msg;
     189             : }
     190             : 
     191             : static char*
     192           3 : commit_action(mvc* m, monetdbe_database_internal *mdbe, monetdbe_result **result, monetdbe_result_internal *res_internal)
     193             : {
     194             :         char *commit_msg = MAL_SUCCEED;
     195             : 
     196             :         /* if an error already exists from MonetDBe set the session status to dirty */
     197           3 :         if (mdbe->msg != MAL_SUCCEED && m->session->tr->active && !m->session->status)
     198           0 :                 m->session->status = -1;
     199           3 :         commit_msg = SQLautocommit(m); /* handle autocommit */
     200             : 
     201           3 :         if (mdbe->msg != MAL_SUCCEED || commit_msg != MAL_SUCCEED) {
     202           0 :                 if (res_internal) {
     203           0 :                         char* other = monetdbe_cleanup_result_internal(mdbe, res_internal);
     204           0 :                         if (other)
     205           0 :                                 freeException(other);
     206             :                 }
     207           0 :                 if (result)
     208           0 :                         *result = NULL;
     209           0 :                 (void)set_error(mdbe, commit_msg);
     210             :         }
     211           3 :         return mdbe->msg;
     212             : }
     213             : 
     214             : static int
     215           1 : validate_database_handle_noerror(monetdbe_database_internal *mdbe)
     216             : {
     217           1 :         if (!monetdbe_embedded_initialized || !MCvalid(mdbe->c))
     218           0 :                 return 0;
     219             :         clear_error(mdbe);
     220           1 :         return 1;
     221             : }
     222             : 
     223             : // Call this function always inside the embedded_lock
     224             : static char*
     225           5 : validate_database_handle(monetdbe_database_internal *mdbe, const char* call)
     226             : {
     227           5 :         if (!monetdbe_embedded_initialized)
     228           0 :                 return createException(MAL, call, "MonetDBe has not yet started");
     229           5 :         if (!MCvalid(mdbe->c))
     230           0 :                 return createException(MAL, call, "Invalid database handle");
     231             :         clear_error(mdbe);
     232           5 :         return MAL_SUCCEED;
     233             : }
     234             : 
     235             : static void
     236           2 : monetdbe_destroy_column(monetdbe_column* column)
     237             : {
     238             :         size_t j;
     239             : 
     240           2 :         if (!column)
     241             :                 return;
     242             : 
     243           2 :         if (column->type == monetdbe_str) {
     244             :                 // FIXME: clean up individual strings
     245           1 :                 char** data = (char**)column->data;
     246           3 :                 for(j = 0; j < column->count; j++) {
     247           2 :                         if (data[j])
     248           2 :                                 GDKfree(data[j]);
     249             :                 }
     250           1 :         } else if (column->type == monetdbe_blob) {
     251           0 :                 monetdbe_data_blob* data = (monetdbe_data_blob*)column->data;
     252           0 :                 for(j = 0; j < column->count; j++) {
     253           0 :                         if (data[j].data)
     254           0 :                                 GDKfree(data[j].data);
     255             :                 }
     256             :         }
     257           2 :         GDKfree(column->data);
     258           2 :         GDKfree(column);
     259             : }
     260             : 
     261             : static char*
     262           1 : monetdbe_cleanup_result_internal(monetdbe_database_internal *mdbe, monetdbe_result_internal* result)
     263             : {
     264           1 :         mvc *m = NULL;
     265             : 
     266           1 :         assert(!result || !result->mdbe || result->mdbe == mdbe);
     267           1 :         if ((mdbe->msg = validate_database_handle(mdbe, "monetdbe.monetdbe_cleanup_result_internal")) != MAL_SUCCEED)
     268             :                 return mdbe->msg;
     269           1 :         if ((mdbe->msg = getSQLContext(mdbe->c, NULL, &m, NULL)) != MAL_SUCCEED)
     270           0 :                 goto cleanup;
     271             : 
     272           1 :         if (result->monetdbe_resultset)
     273           1 :                 res_tables_destroy(result->monetdbe_resultset);
     274             : 
     275           1 :         if (result->converted_columns) {
     276           3 :                 for (size_t i = 0; i < result->res.ncols; i++)
     277           2 :                         monetdbe_destroy_column(result->converted_columns[i]);
     278           1 :                 GDKfree(result->converted_columns);
     279             :         }
     280           1 :         GDKfree(result);
     281           1 : cleanup:
     282           1 :         return commit_action(m, mdbe, NULL, NULL);
     283             : }
     284             : 
     285             : static char*
     286           1 : monetdbe_get_results(monetdbe_result** result, monetdbe_database_internal *mdbe) {
     287             : 
     288           1 :         backend *be = NULL;
     289             : 
     290           1 :         *result = NULL;
     291           1 :         if ((mdbe->msg = getBackendContext(mdbe->c, &be)) != NULL)
     292             :                 return mdbe->msg;
     293             : 
     294           1 :         mvc *m = be->mvc;
     295             :         monetdbe_result_internal* res_internal;
     296             : 
     297           1 :         if (!(res_internal = GDKzalloc(sizeof(monetdbe_result_internal)))) {
     298           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_results", MAL_MALLOC_FAIL));
     299           0 :                 return mdbe->msg;
     300             :         }
     301             :         // TODO: set type of result outside.
     302           1 :         res_internal->res.last_id = be->last_id;
     303           1 :         res_internal->mdbe = mdbe;
     304           1 :         *result = (monetdbe_result*) res_internal;
     305           1 :         m->reply_size = -2; /* do not clean up result tables */
     306             : 
     307           1 :         if (be->results) {
     308           1 :                 res_internal->res.ncols = (size_t) be->results->nr_cols;
     309           1 :                 res_internal->monetdbe_resultset = be->results;
     310           1 :                 if (be->results->nr_cols > 0)
     311           1 :                         res_internal->res.nrows = be->results->nr_rows;
     312           1 :                 be->results = NULL;
     313           1 :                 res_internal->converted_columns = GDKzalloc(sizeof(monetdbe_column*) * res_internal->res.ncols);
     314           1 :                 if (!res_internal->converted_columns) {
     315           0 :                         GDKfree(res_internal);
     316           0 :                         *result = NULL;
     317           0 :                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_results", MAL_MALLOC_FAIL));
     318           0 :                         return mdbe->msg;
     319             :                 }
     320             :         }
     321             : 
     322             :         return MAL_SUCCEED;
     323             : }
     324             : 
     325             : static char*
     326           0 : monetdbe_query_internal(monetdbe_database_internal *mdbe, char* query, monetdbe_result** result, monetdbe_cnt* affected_rows, int *prepare_id, char language)
     327             : {
     328             :         char *nq = NULL;
     329           0 :         Client c = mdbe->c;
     330           0 :         mvc* m = NULL;
     331             :         backend *b;
     332             :         size_t query_len, input_query_len, prep_len = 0;
     333             :         buffer query_buf;
     334             :         stream *query_stream = NULL;
     335           0 :         bstream *old_bstream = c->fdin;
     336           0 :         stream *fdout = c->fdout;
     337             :         bool fdin_changed = false;
     338             : 
     339           0 :         if (result)
     340           0 :                 *result = NULL;
     341             : 
     342           0 :         if ((mdbe->msg = validate_database_handle(mdbe, "monetdbe.monetdbe_query_internal")) != MAL_SUCCEED)
     343             :                 return mdbe->msg;
     344             : 
     345           0 :         if ((mdbe->msg = getSQLContext(c, NULL, &m, NULL)) != MAL_SUCCEED)
     346           0 :                 goto cleanup;
     347           0 :         b = (backend *) c->sqlcontext;
     348             : 
     349           0 :         if (!query) {
     350           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_query_internal", "Query missing"));
     351           0 :                 goto cleanup;
     352             :         }
     353           0 :         if (!(query_stream = buffer_rastream(&query_buf, "sqlstatement"))) {
     354           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_query_internal", "Could not setup query stream"));
     355           0 :                 goto cleanup;
     356             :         }
     357           0 :         input_query_len = strlen(query);
     358           0 :         query_len = input_query_len + 3;
     359           0 :         if (prepare_id) {
     360             :                 prep_len = sizeof("PREPARE ")-1;
     361           0 :                 query_len += prep_len;
     362             :         }
     363           0 :         if (!(nq = GDKmalloc(query_len))) {
     364           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_query_internal", MAL_MALLOC_FAIL));
     365           0 :                 goto cleanup;
     366             :         }
     367           0 :         if (prepare_id)
     368           0 :                 strcpy(nq, "PREPARE ");
     369           0 :         strcpy(nq + prep_len, query);
     370           0 :         strcpy(nq + prep_len + input_query_len, "\n;");
     371             : 
     372           0 :         query_buf.pos = 0;
     373           0 :         query_buf.len = query_len;
     374           0 :         query_buf.buf = nq;
     375             : 
     376             :         fdin_changed = true;
     377           0 :         if (!(c->fdin = bstream_create(query_stream, query_len))) {
     378           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_query_internal", "Could not setup query stream"));
     379           0 :                 goto cleanup;
     380             :         }
     381             :         query_stream = NULL;
     382           0 :         if (bstream_next(c->fdin) < 0) {
     383           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_query_internal", "Internal error while starting the query"));
     384           0 :                 goto cleanup;
     385             :         }
     386             : 
     387           0 :         assert(language);
     388           0 :         b->language = language;
     389           0 :         b->output_format = OFMT_NONE;
     390           0 :         b->no_mitosis = 0;
     391           0 :         m->user_id = m->role_id = USER_MONETDB;
     392           0 :         m->errstr[0] = '\0';
     393           0 :         m->params = NULL;
     394           0 :         m->sym = NULL;
     395           0 :         m->label = 0;
     396           0 :         if (m->sa)
     397           0 :                 m->sa = sa_reset(m->sa);
     398           0 :         m->scanner.mode = LINE_N;
     399           0 :         m->scanner.rs = c->fdin;
     400           0 :         scanner_query_processed(&(m->scanner));
     401             : 
     402           0 :         if ((mdbe->msg = MSinitClientPrg(c, "user", "main")) != MAL_SUCCEED)
     403           0 :                 goto cleanup;
     404           0 :         if (prepare_id)
     405           0 :                 m->emode = m_prepare;
     406           0 :         if ((mdbe->msg = SQLparser(c)) != MAL_SUCCEED)
     407           0 :                 goto cleanup;
     408           0 :         if (m->emode == m_prepare && prepare_id)
     409           0 :                 *prepare_id = b->q->id;
     410           0 :         c->fdout = NULL;
     411           0 :         if ((mdbe->msg = SQLengine(c)) != MAL_SUCCEED)
     412           0 :                 goto cleanup;
     413           0 :         if (!b->results && b->rowcnt >= 0 && affected_rows)
     414           0 :                 *affected_rows = b->rowcnt;
     415             : 
     416           0 :         if (result) {
     417           0 :                 if ((mdbe->msg = monetdbe_get_results(result, mdbe)) != MAL_SUCCEED) {
     418           0 :                         goto cleanup;
     419             :                 }
     420             : 
     421           0 :                 if (m->emode & m_prepare)
     422           0 :                         (*(monetdbe_result_internal**) result)->type = Q_PREPARE;
     423             :                 else
     424           0 :                         (*(monetdbe_result_internal**) result)->type = (b->results) ? b->results->query_type : m->type;
     425             :         }
     426             : 
     427           0 : cleanup:
     428           0 :         c->fdout = fdout;
     429           0 :         if (nq)
     430           0 :                 GDKfree(nq);
     431           0 :         MSresetInstructions(c->curprg->def, 1);
     432           0 :         if (fdin_changed) { //c->fdin was set
     433           0 :                 bstream_destroy(c->fdin);
     434           0 :                 c->fdin = old_bstream;
     435             :         }
     436           0 :         if (query_stream)
     437           0 :                 close_stream(query_stream);
     438             : 
     439           0 :         return commit_action(m, mdbe, result, result?*(monetdbe_result_internal**) result:NULL);
     440             : }
     441             : 
     442             : static int
     443           1 : monetdbe_close_remote(monetdbe_database_internal *mdbe)
     444             : {
     445           1 :         assert(mdbe && mdbe->mid);
     446             : 
     447             :         int err = 0;
     448             : 
     449           1 :         if (mdbe->msg) {
     450             :                 err = 1;
     451             :                 clear_error(mdbe);
     452             :         }
     453             : 
     454           1 :         if ( (mdbe->msg = RMTdisconnect(NULL, &(mdbe->mid))) != MAL_SUCCEED) {
     455             :                 err = 1;
     456             :                 clear_error(mdbe);
     457             :         }
     458             : 
     459           1 :         GDKfree(mdbe->mid);
     460           1 :         mdbe->mid = NULL;
     461             : 
     462           1 :         return err;
     463             : }
     464             : 
     465             : static int
     466           1 : monetdbe_close_internal(monetdbe_database_internal *mdbe)
     467             : {
     468           1 :         assert(mdbe);
     469             : 
     470           1 :         if (validate_database_handle_noerror(mdbe)) {
     471           1 :                 open_dbs--;
     472           1 :                 char *msg = SQLexitClient(mdbe->c);
     473           1 :                 if (msg)
     474           0 :                         freeException(msg);
     475           1 :                 MCcloseClient(mdbe->c);
     476             :         }
     477           1 :         GDKfree(mdbe);
     478           1 :         return 0;
     479             : }
     480             : 
     481             : static int
     482           2 : monetdbe_workers_internal(monetdbe_database_internal *mdbe, monetdbe_options *opts)
     483             : {
     484             :         int workers = 0;
     485           2 :         if (opts && opts->nr_threads) {
     486           0 :                 if (opts->nr_threads < 0)
     487           0 :                         set_error(mdbe,createException(MAL, "monetdbe.monetdbe_startup", "Nr_threads should be positive"));
     488             :                 else
     489           0 :                         workers = GDKnr_threads = opts->nr_threads;
     490             :         }
     491           2 :         return workers;
     492             : }
     493             : 
     494             : static int
     495           2 : monetdbe_memory_internal(monetdbe_database_internal *mdbe, monetdbe_options *opts)
     496             : {
     497             :         int memory = 0;
     498           2 :         if (opts && opts->memorylimit) {
     499           0 :                 if (opts->memorylimit < 0)
     500           0 :                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", "Memorylimit should be positive"));
     501             :                 else /* Memory limit is session specific */
     502             :                         memory = opts->memorylimit;
     503             :         }
     504           2 :         return memory;
     505             : }
     506             : 
     507             : static int
     508           1 : monetdbe_querytimeout_internal(monetdbe_database_internal *mdbe, monetdbe_options *opts)
     509             : {
     510             :         int querytimeout = 0;
     511           1 :         if (opts && opts->querytimeout) {
     512           0 :                 if (opts->querytimeout < 0)
     513           0 :                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", "Query timeout should be positive (in sec)"));
     514             :                 else
     515             :                         querytimeout = opts->querytimeout;
     516             :         }
     517           1 :         return querytimeout;
     518             : }
     519             : 
     520             : static int
     521           1 : monetdbe_sessiontimeout_internal(monetdbe_database_internal *mdbe, monetdbe_options *opts)
     522             : {
     523             :         int sessiontimeout = 0;
     524           1 :         if (opts && opts->sessiontimeout) {
     525           0 :                 if (opts->sessiontimeout < 0)
     526           0 :                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", "Session timeout should be positive (in sec)"));
     527             :                 else
     528             :                         sessiontimeout = opts->sessiontimeout;
     529             :         }
     530           1 :         return sessiontimeout;
     531             : }
     532             : 
     533             : static int
     534           1 : monetdbe_open_internal(monetdbe_database_internal *mdbe, monetdbe_options *opts )
     535             : {
     536             :         mvc *m;
     537             : 
     538           1 :         if (!mdbe)
     539             :                 return -1;
     540           1 :         if (!monetdbe_embedded_initialized) {
     541           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_open_internal", "Embedded MonetDB is not started"));
     542           0 :                 goto cleanup;
     543             :         }
     544           1 :         mdbe->c = MCinitClient((oid) 0, 0, 0);
     545           1 :         if (!MCvalid(mdbe->c)) {
     546           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_open_internal", "Failed to initialize client"));
     547           0 :                 goto cleanup;
     548             :         }
     549           1 :         mdbe->c->curmodule = mdbe->c->usermodule = userModule();
     550           1 :         mdbe->c->workerlimit = monetdbe_workers_internal(mdbe, opts);
     551           1 :         mdbe->c->memorylimit = monetdbe_memory_internal(mdbe, opts);
     552           1 :         mdbe->c->querytimeout = monetdbe_querytimeout_internal(mdbe, opts);
     553           1 :         mdbe->c->sessiontimeout = monetdbe_sessiontimeout_internal(mdbe, opts);
     554           1 :         if (mdbe->msg)
     555           0 :                 goto cleanup;
     556           1 :         if (mdbe->c->usermodule == NULL) {
     557           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_open_internal", "Failed to initialize client MAL module"));
     558           0 :                 goto cleanup;
     559             :         }
     560           1 :         if ((mdbe->msg = SQLinitClient(mdbe->c)) != MAL_SUCCEED ||
     561           1 :                 (mdbe->msg = getSQLContext(mdbe->c, NULL, &m, NULL)) != MAL_SUCCEED)
     562           0 :                 goto cleanup;
     563           1 :         m->session->auto_commit = 1;
     564           1 :         if (!m->pa)
     565           0 :                 m->pa = sa_create(NULL);
     566           1 :         if (!m->sa)
     567           1 :                 m->sa = sa_create(m->pa);
     568           1 :         if (!m->ta)
     569           0 :                 m->ta = sa_create(m->pa);
     570           1 :         if (!m->pa || !m->sa || !m->ta) {
     571           0 :                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_open_internal", MAL_MALLOC_FAIL));
     572           0 :                 goto cleanup;
     573             :         }
     574           1 : cleanup:
     575           1 :         if (mdbe->msg)
     576             :                 return -2;
     577           1 :         mdbe->blob_null.data = NULL;
     578           1 :         data_from_date(date_nil, &mdbe->date_null);
     579           1 :         data_from_time(daytime_nil, &mdbe->time_null);
     580           1 :         data_from_timestamp(timestamp_nil, &mdbe->timestamp_null);
     581           1 :         open_dbs++;
     582           1 :         return 0;
     583             : }
     584             : 
     585             : static void
     586           1 : monetdbe_shutdown_internal(void) // Call this function always inside the embedded_lock
     587             : {
     588           1 :         if (monetdbe_embedded_initialized && (open_dbs == 0)) {
     589           1 :                 malEmbeddedReset();
     590           1 :                 monetdbe_embedded_initialized = false;
     591           1 :                 if (monetdbe_embedded_url)
     592           0 :                         GDKfree(monetdbe_embedded_url);
     593           1 :                 monetdbe_embedded_url = NULL;
     594             :         }
     595           1 : }
     596             : 
     597             : static void
     598           1 : monetdbe_startup(monetdbe_database_internal *mdbe, const char* dbdir, monetdbe_options *opts)
     599             : {
     600             :         // Only call monetdbe_startup when there is no monetdb internal yet initialized.
     601           1 :         assert(!monetdbe_embedded_initialized);
     602             : 
     603           1 :         opt *set = NULL;
     604             :         int setlen;
     605             :         bool with_mapi_server;
     606             :         int workers, memory;
     607             :         gdk_return gdk_res;
     608             : 
     609           1 :         GDKfataljumpenable = 1;
     610             : 
     611           1 :         if(setjmp(GDKfataljump) != 0) {
     612           0 :                 assert(0);
     613             :                 mdbe->msg = GDKfatalmsg;
     614             :                 // we will get here if GDKfatal was called.
     615             :                 if (mdbe->msg == NULL)
     616             :                         mdbe->msg = createException(MAL, "monetdbe.monetdbe_startup", "GDKfatal() with unspecified error");
     617             :                 goto cleanup;
     618             :         }
     619             : 
     620             :          with_mapi_server = false;
     621             : 
     622           1 :         if (monetdbe_embedded_initialized) {
     623           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", "MonetDBe is already initialized"));
     624           0 :                 return;
     625             :         }
     626             : 
     627           1 :         if ((setlen = mo_builtin_settings(&set)) == 0) {
     628           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", MAL_MALLOC_FAIL));
     629           0 :                 goto cleanup;
     630             :         }
     631           1 :         if (dbdir && (setlen = mo_add_option(&set, setlen, opt_cmdline, "gdk_dbpath", dbdir)) == 0) {
     632           0 :                 mo_free_options(set, setlen);
     633           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", MAL_MALLOC_FAIL));
     634           0 :                 goto cleanup;
     635             :         }
     636           1 :         if (opts && opts->nr_threads == 1)
     637           0 :                 setlen = mo_add_option(&set, setlen, opt_cmdline, "sql_optimizer", "sequential_pipe");
     638             :         else
     639           1 :                 setlen = mo_add_option(&set, setlen, opt_cmdline, "sql_optimizer", "default_pipe");
     640             : 
     641           1 :         if (setlen == 0) {
     642           0 :                 mo_free_options(set, setlen);
     643           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", MAL_MALLOC_FAIL));
     644           0 :                 goto cleanup;
     645             :         }
     646             : 
     647           1 :         if (opts && opts->mapi_server) {
     648             :                 /*This monetdbe instance wants to listen to external mapi client connections.*/
     649             :                 with_mapi_server = true;
     650           0 :                 if (opts->mapi_server->port) {
     651             :                         int psetlen = setlen;
     652           0 :                         setlen = mo_add_option(&set, setlen, opt_cmdline, "mapi_port", opts->mapi_server->port);
     653           0 :                         if (setlen == psetlen) {
     654           0 :                                 mo_free_options(set, setlen);
     655           0 :                                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", MAL_MALLOC_FAIL));
     656           0 :                                 goto cleanup;
     657             :                         }
     658             :                 }
     659           0 :                 if (opts->mapi_server->usock) {
     660             :                         int psetlen = setlen;
     661           0 :                         setlen = mo_add_option(&set, setlen, opt_cmdline, "mapi_usock", opts->mapi_server->usock);
     662           0 :                         if (setlen == psetlen) {
     663           0 :                                 mo_free_options(set, setlen);
     664           0 :                                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", MAL_MALLOC_FAIL));
     665           0 :                                 goto cleanup;
     666             :                         }
     667             :                 }
     668             :         }
     669             : 
     670             :         /* set the output of GDKtracer logs */
     671           1 :         if (opts && opts->trace_file) {
     672             :                 /* if file specified, use it */
     673           0 :                 if (GDKtracer_set_tracefile(opts->trace_file) != GDK_SUCCEED) {
     674           0 :                         mo_free_options(set, setlen);
     675           0 :                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", GDK_EXCEPTION));
     676           0 :                         goto cleanup;
     677             :                 }
     678           0 :                 GDKtracer_set_adapter("BASIC");
     679             :         } else {
     680             :                 /* otherwise no trace output */
     681           1 :                 GDKtracer_set_adapter("MBEDDED");
     682             :         }
     683             : 
     684           1 :         workers = monetdbe_workers_internal(mdbe, opts);
     685           1 :         memory = monetdbe_memory_internal(mdbe, opts);
     686           1 :         if (memory)
     687           0 :                         GDK_vm_maxsize = (size_t) memory << 20; /* convert from MiB to bytes */
     688           1 :         if (mdbe->msg)
     689           0 :                         goto cleanup;
     690             : 
     691           1 :         if (!dbdir) { /* in-memory */
     692           1 :                 if (BBPaddfarm(NULL, (1U << PERSISTENT) | (1U << TRANSIENT), false) != GDK_SUCCEED) {
     693           0 :                         mo_free_options(set, setlen);
     694           0 :                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", "Cannot add in-memory farm"));
     695           0 :                         goto cleanup;
     696             :                 }
     697             :         } else {
     698           0 :                 if (BBPaddfarm(dbdir, 1U << PERSISTENT, false) != GDK_SUCCEED ||
     699           0 :                         BBPaddfarm(/*dbextra ? dbextra : */dbdir, 1U << TRANSIENT, false) != GDK_SUCCEED) {
     700           0 :                         mo_free_options(set, setlen);
     701           0 :                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", "Cannot add farm %s", dbdir));
     702           0 :                         goto cleanup;
     703             :                 }
     704           0 :                 if (GDKcreatedir(dbdir) != GDK_SUCCEED) {
     705           0 :                         mo_free_options(set, setlen);
     706           0 :                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", "Cannot create directory %s", dbdir));
     707           0 :                         goto cleanup;
     708             :                 }
     709             :         }
     710           1 :         gdk_res = GDKinit(set, setlen, true);
     711           1 :         mo_free_options(set, setlen);
     712           1 :         if (gdk_res != GDK_SUCCEED) {
     713           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", "GDKinit() failed"));
     714           0 :                 goto cleanup;
     715             :         }
     716             : 
     717           1 :         if ((mdbe->msg = malEmbeddedBoot(workers, memory, 0, 0, with_mapi_server)) != MAL_SUCCEED)
     718           0 :                 goto cleanup;
     719             : 
     720           1 :         monetdbe_embedded_initialized = true;
     721           1 :         monetdbe_embedded_url = dbdir?GDKstrdup(dbdir):NULL;
     722           1 :         if (dbdir && !monetdbe_embedded_url)
     723           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_startup", MAL_MALLOC_FAIL));
     724           1 :         GDKfataljumpenable = 0;
     725           1 : cleanup:
     726           1 :         if (mdbe->msg)
     727           0 :                 monetdbe_shutdown_internal();
     728             : }
     729             : 
     730           0 : static bool urls_matches(const char* l, const char* r) {
     731           0 :         return (l && r && (strcmp(l, r) == 0)) || (l == NULL && r == NULL);
     732             : }
     733             : 
     734             : static inline str
     735           1 : monetdbe_create_uri(const char* host, const int port, const char* database) {
     736             :         const char* protocol = "mapi:monetdb://";
     737             : 
     738             :         const size_t sl_protocol = strlen(protocol);
     739           1 :         const size_t sl_host = strlen(host);
     740             :         const size_t sl_max_port = 6; // 2^16-1 < 100 000 = 10^5, i.e. always less then 6 digits.
     741           1 :         const size_t sl_database = strlen(database);
     742           1 :         const size_t sl_total = sl_protocol + sl_host + 1 /* : */ + sl_max_port + 1 + /* / */ + sl_database;
     743             : 
     744           1 :         str uri_buffer = GDKmalloc(sl_total + 1 /* terminator */);
     745           1 :         if (!uri_buffer)
     746             :                 return NULL;
     747             : 
     748           1 :         snprintf(uri_buffer, sl_total, "%s%s:%d/%s", protocol, host, port, database);
     749             : 
     750           1 :         return uri_buffer;
     751             : }
     752             : 
     753             : static int
     754           1 : monetdbe_open_remote(monetdbe_database_internal *mdbe, monetdbe_options *opts) {
     755           1 :         assert(opts);
     756             : 
     757           1 :         monetdbe_remote* remote = opts->remote;
     758           1 :         if (!remote) {
     759           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_open_remote", "Missing remote proxy settings"));
     760           0 :                 return -1;
     761             :         }
     762             : 
     763           1 :         Client c = mdbe->c;
     764             : 
     765           1 :         assert(!c->curprg);
     766             : 
     767             :         const char *mod = "user";
     768             :         char nme[16];
     769           1 :         const char *name = number2name(nme, sizeof(nme), ++((backend*)  c->sqlcontext)->remote);
     770           1 :         c->curprg = newFunction(putName(mod), putName(name), FUNCTIONsymbol);
     771             : 
     772           1 :         if (c->curprg == NULL) {
     773           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_open_remote", MAL_MALLOC_FAIL));
     774           0 :                 return -2;
     775             :         }
     776             : 
     777             :         char* url;
     778           1 :         if ((url = monetdbe_create_uri(remote->host, remote->port, remote->database)) == NULL) {
     779           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_open_remote", MAL_MALLOC_FAIL));
     780           0 :                 return -2;
     781             :         }
     782             : 
     783           1 :         MalBlkPtr mb = c->curprg->def;
     784             : 
     785           1 :         InstrPtr q = getInstrPtr(mb, 0);
     786           1 :         q->argc = q->retc = 0;
     787           1 :         q = pushReturn(mb, q, newTmpVariable(mb, TYPE_str));
     788             : 
     789           1 :         InstrPtr p = newFcnCall(mb, remoteRef, connectRef);
     790           1 :         p = pushStr(mb, p, url);
     791           1 :         p = pushStr(mb, p, remote->username);
     792           1 :         p = pushStr(mb, p, remote->password);
     793           1 :         p = pushStr(mb, p, "msql");
     794           1 :         p = pushBit(mb, p, 1);
     795             : 
     796           1 :         GDKfree(url);
     797             :         url = NULL;
     798             : 
     799           1 :         q = newInstruction(mb, NULL, NULL);
     800           1 :         q->barrier= RETURNsymbol;
     801           1 :         q = pushReturn(mb, q, getArg(p, 0));
     802             : 
     803           1 :         pushInstruction(mb, q);
     804             : 
     805             :         if (p == NULL) {
     806             :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_open_remote", MAL_MALLOC_FAIL));
     807             :                 freeSymbol(c->curprg);
     808             :                 c->curprg= NULL;
     809             :                 return -2;
     810             :         }
     811           1 :         if ( (mdbe->msg = chkProgram(c->usermodule, mb)) != MAL_SUCCEED ) {
     812           0 :                 freeSymbol(c->curprg);
     813           0 :                 c->curprg= NULL;
     814           0 :                 return -2;
     815             :         }
     816           1 :         MalStkPtr stk = prepareMALstack(mb, mb->vsize);
     817           1 :         if (!stk) {
     818           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_open_remote", MAL_MALLOC_FAIL));
     819           0 :                 freeSymbol(c->curprg);
     820           0 :                 c->curprg= NULL;
     821           0 :                 return -2;
     822             :         }
     823           1 :         stk->keepAlive = TRUE;
     824           1 :         if ( (mdbe->msg = runMALsequence(c, mb, 1, 0, stk, 0, 0)) != MAL_SUCCEED ) {
     825           0 :                 freeStack(stk);
     826           0 :                 freeSymbol(c->curprg);
     827           0 :                 c->curprg= NULL;
     828           0 :                 return -2;
     829             :         }
     830             : 
     831           1 :         if ((mdbe->mid = GDKstrdup(*getArgReference_str(stk, p, 0))) == NULL) {
     832           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_open_remote", MAL_MALLOC_FAIL));
     833           0 :                 freeStack(stk);
     834           0 :                 freeSymbol(c->curprg);
     835           0 :                 c->curprg= NULL;
     836           0 :                 return -2;
     837             :         }
     838             : 
     839           1 :         garbageCollector(c, mb, stk, TRUE);
     840           1 :         freeStack(stk);
     841             : 
     842           1 :         freeSymbol(c->curprg);
     843           1 :         c->curprg= NULL;
     844             : 
     845           1 :         return 0;
     846             : }
     847             : 
     848             : int
     849           1 : monetdbe_open(monetdbe_database *dbhdl, char *url, monetdbe_options *opts)
     850             : {
     851             :         int res = 0;
     852             : 
     853           1 :         if (!dbhdl)
     854             :                 return -1;
     855           1 :         if (url &&
     856           0 :                 (strcmp(url, "in-memory") == 0 ||
     857           0 :                  /* backward compatibility: */ strcmp(url, ":memory:") == 0))
     858             :                 url = NULL;
     859           1 :         MT_lock_set(&embedded_lock);
     860           1 :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)GDKzalloc(sizeof(monetdbe_database_internal));
     861           1 :         if (!mdbe) {
     862           0 :                 MT_lock_unset(&embedded_lock);
     863           0 :                 return -1;
     864             :         }
     865           1 :         *dbhdl = (monetdbe_database)mdbe;
     866           1 :         mdbe->msg = NULL;
     867           1 :         mdbe->c = NULL;
     868             : 
     869           1 :         bool is_remote = (opts && (opts->remote != NULL));
     870           1 :         if (!monetdbe_embedded_initialized) {
     871             :                 /* When used as a remote mapi proxy,
     872             :                  * it is still necessary to have an initialized monetdbe. E.g. for BAT life cycle management.
     873             :                  * Use an ephemeral/anonymous dbfarm when there is no initialized monetdbe yet.
     874             :                  */
     875           1 :                 assert(!is_remote||url==NULL);
     876           1 :                 monetdbe_startup(mdbe, url, opts);
     877           0 :         } else if (!is_remote && !urls_matches(monetdbe_embedded_url, url)) {
     878           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_open", "monetdbe_open currently only one active database is supported"));
     879             :         }
     880           1 :         if (!mdbe->msg)
     881           1 :                 res = monetdbe_open_internal(mdbe, opts);
     882             : 
     883           1 :         if (res == 0 && is_remote)
     884           1 :                 res = monetdbe_open_remote(mdbe, opts);
     885             : 
     886           1 :         MT_lock_unset(&embedded_lock);
     887           1 :         if (mdbe->msg)
     888           0 :                 return -2;
     889             :         return res;
     890             : }
     891             : 
     892             : int
     893           1 : monetdbe_close(monetdbe_database dbhdl)
     894             : {
     895           1 :         if (!dbhdl)
     896             :                 return 0;
     897             : 
     898             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
     899             : 
     900             :         int err = 0;
     901             : 
     902           1 :         MT_lock_set(&embedded_lock);
     903           1 :         if (mdbe->mid)
     904           1 :                 err = monetdbe_close_remote(mdbe);
     905             : 
     906           1 :         err = (monetdbe_close_internal(mdbe) || err);
     907             : 
     908           1 :         if (!open_dbs)
     909           1 :                 monetdbe_shutdown_internal();
     910           1 :         MT_lock_unset(&embedded_lock);
     911             : 
     912           1 :         if (err) {
     913           0 :                 return -2;
     914             :         }
     915             : 
     916             :         return 0;
     917             : }
     918             : 
     919             : char *
     920           0 : monetdbe_error(monetdbe_database dbhdl)
     921             : {
     922           0 :         if (!dbhdl)
     923             :                 return NULL;
     924             : 
     925             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
     926           0 :         return mdbe->msg;
     927             : }
     928             : 
     929             : char*
     930           0 : monetdbe_dump_database(monetdbe_database dbhdl, const char *filename)
     931             : {
     932           0 :         if (!dbhdl)
     933             :                 return NULL;
     934             : 
     935             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
     936             : 
     937           0 :         if (mdbe->mid) {
     938           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_dump_database", PROGRAM_NYI));
     939           0 :                 return mdbe->msg;
     940             :         }
     941             : 
     942           0 :         if ((mdbe->msg = validate_database_handle(mdbe, "embedded.monetdbe_dump_database")) != MAL_SUCCEED) {
     943             :                 return mdbe->msg;
     944             :         }
     945             : 
     946           0 :         mdbe->msg = monetdbe_mapi_dump_database(dbhdl, filename);
     947             : 
     948           0 :         return mdbe->msg;
     949             : }
     950             : 
     951             : char*
     952           0 : monetdbe_dump_table(monetdbe_database dbhdl, const char *sname, const char *tname, const char *filename)
     953             : {
     954           0 :         if (!dbhdl)
     955             :                 return NULL;
     956             : 
     957             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
     958             : 
     959           0 :         if (mdbe->mid) {
     960           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_dump_database", PROGRAM_NYI));
     961           0 :                 return mdbe->msg;
     962             :         }
     963             : 
     964           0 :         if ((mdbe->msg = validate_database_handle(mdbe, "embedded.monetdbe_dump_table")) != MAL_SUCCEED) {
     965             : 
     966             :                 return mdbe->msg;
     967             :         }
     968             : 
     969           0 :         mdbe->msg = monetdbe_mapi_dump_table(dbhdl, sname, tname, filename);
     970             : 
     971           0 :         return mdbe->msg;
     972             : }
     973             : 
     974             : char*
     975           0 : monetdbe_get_autocommit(monetdbe_database dbhdl, int* result)
     976             : {
     977           0 :         if (!dbhdl)
     978             :                 return NULL;
     979             : 
     980             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
     981             : 
     982           0 :         if ((mdbe->msg = validate_database_handle(mdbe, "monetdbe.monetdbe_get_autocommit")) != MAL_SUCCEED) {
     983             : 
     984             :                 return mdbe->msg;
     985             :         }
     986             : 
     987           0 :         if (!result) {
     988           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_autocommit", "Parameter result is NULL"));
     989             :         } else {
     990           0 :                 mvc *m = ((backend *) mdbe->c->sqlcontext)->mvc;
     991           0 :                 *result = m->session->auto_commit;
     992             :         }
     993             : 
     994           0 :         return mdbe->msg;
     995             : }
     996             : 
     997             : char*
     998           0 : monetdbe_set_autocommit(monetdbe_database dbhdl, int value)
     999             : {
    1000           0 :         if (!dbhdl)
    1001             :                 return NULL;
    1002             : 
    1003             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
    1004             : 
    1005           0 :         if (!validate_database_handle_noerror(mdbe)) {
    1006             : 
    1007             :                 return 0;
    1008             :         }
    1009             : 
    1010           0 :         mvc *m = ((backend *) mdbe->c->sqlcontext)->mvc;
    1011           0 :         int commit = !m->session->auto_commit && value;
    1012             : 
    1013           0 :         m->session->auto_commit = value;
    1014           0 :         m->session->ac_on_commit = m->session->auto_commit;
    1015           0 :         if (m->session->tr->active) {
    1016           0 :                 if (commit) {
    1017           0 :                         mdbe->msg = mvc_commit(m, 0, NULL, true);
    1018             :                 } else {
    1019           0 :                         mdbe->msg = mvc_rollback(m, 0, NULL, true);
    1020             :                 }
    1021             :         }
    1022             : 
    1023           0 :         return mdbe->msg;
    1024             : }
    1025             : 
    1026             : int
    1027           0 : monetdbe_in_transaction(monetdbe_database dbhdl)
    1028             : {
    1029           0 :         if (!dbhdl)
    1030             :                 return 0;
    1031             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
    1032             : 
    1033           0 :         if (!validate_database_handle_noerror(mdbe)) {
    1034             : 
    1035             :                 return 0;
    1036             :         }
    1037             : 
    1038           0 :         mvc *m = ((backend *) mdbe->c->sqlcontext)->mvc;
    1039             :         int result = 0;
    1040             : 
    1041           0 :         if (m->session->tr)
    1042           0 :                 result = m->session->tr->active;
    1043             : 
    1044             :         return result;
    1045             : }
    1046             : 
    1047             : struct callback_context {
    1048             :         monetdbe_database_internal *mdbe;
    1049             : };
    1050             : 
    1051             : static str
    1052           1 : monetdbe_set_remote_results(backend *be, char* tblname, columnar_result* results, size_t nr_results) {
    1053             : 
    1054             :         char* error = NULL;
    1055             : 
    1056           1 :         if (nr_results == 0)
    1057             :                 return MAL_SUCCEED; // No work to do.
    1058             : 
    1059           1 :         BAT* b_0 = BATdescriptor(results[0].id); // Fetch the first column to get the count.
    1060           1 :         if (!b_0) {
    1061           0 :                 error = createException(MAL,"monetdbe.monetdbe_set_remote_results",SQLSTATE(HY005) "Cannot access column descriptor ");
    1062           0 :                 return error;
    1063             :         }
    1064             : 
    1065           1 :         BAT* order = BATdense(0, 0, BATcount(b_0));
    1066           1 :         if (!order) {
    1067           0 :                 BBPunfix(b_0->batCacheid);
    1068           0 :                 error = createException(MAL,"monetdbe.monetdbe_set_remote_results",SQLSTATE(HY005) MAL_MALLOC_FAIL);
    1069           0 :                 return error;
    1070             :         }
    1071             : 
    1072           1 :         int res = mvc_result_table(be, 0, (int) nr_results, Q_TABLE, order);
    1073           1 :         BBPunfix(order->batCacheid);
    1074           1 :         if (res < 0) {
    1075           0 :                 BBPunfix(b_0->batCacheid);
    1076           0 :                 error = createException(MAL,"monetdbe.monetdbe_set_remote_results",SQLSTATE(HY005) "Cannot create result table");
    1077           0 :                 return error;
    1078             :         }
    1079             : 
    1080           3 :         for (size_t i = 0; i < nr_results; i++) {
    1081             :                 BAT *b = NULL;
    1082           2 :                 if (i > 0)
    1083           1 :                         b = BATdescriptor(results[i].id);
    1084             :                 else
    1085             :                         b = b_0; // We already fetched this first column
    1086             : 
    1087           2 :                 char* colname   = results[i].colname;
    1088           2 :                 char* tpename   = results[i].tpename;
    1089           2 :                 int digits              = results[i].digits;
    1090           2 :                 int scale               = results[i].scale;
    1091             : 
    1092           2 :                 if (b == NULL) {
    1093           0 :                         error = createException(MAL,"monetdbe.monetdbe_result_cb",SQLSTATE(HY005) "Cannot access column descriptor");
    1094           0 :                         break;
    1095             :                 }
    1096             : 
    1097           2 :                 int res = mvc_result_column(be, tblname, colname, tpename, digits, scale, b);
    1098           2 :                 BBPunfix(b->batCacheid);
    1099           2 :                 if (res) {
    1100           0 :                         error = createException(MAL,"monetdbe.monetdbe_result_cb", SQLSTATE(42000) "Cannot access column descriptor %s.%s",tblname,colname);
    1101           0 :                         break;
    1102             :                 }
    1103             :         }
    1104             : 
    1105           1 :         if (error)
    1106           0 :                 res_tables_destroy(be->results);
    1107             :         return error;
    1108             : }
    1109             : 
    1110             : static str
    1111           1 : monetdbe_result_cb(void* context, char* tblname, columnar_result* results, size_t nr_results) {
    1112           1 :         monetdbe_database_internal *mdbe = ((struct callback_context*) context)->mdbe;
    1113             : 
    1114           1 :         backend *be = NULL;
    1115           1 :         if ((mdbe->msg = getBackendContext(mdbe->c, &be)) != NULL)
    1116             :                 return mdbe->msg;
    1117             : 
    1118           1 :         return monetdbe_set_remote_results(be, tblname, results, nr_results);
    1119             : }
    1120             : 
    1121             : struct prepare_callback_context {
    1122             :         int* prepare_id;
    1123             :         monetdbe_database_internal *mdbe;
    1124             : };
    1125             : 
    1126             : static str
    1127           0 : monetdbe_prepare_cb(void* context, char* tblname, columnar_result* results, size_t nr_results) {
    1128             :         (void) tblname;
    1129           0 :         monetdbe_database_internal *mdbe        = ((struct prepare_callback_context*) context)->mdbe;
    1130           0 :         int *prepare_id                                         = ((struct prepare_callback_context*) context)->prepare_id;
    1131             : 
    1132           0 :         if (nr_results != 7) // 1) btype 2) bdigits 3) bscale 4) bschema 5) btable 6) bcolumn 7) bimpl
    1133           0 :                 return createException(SQL, "monetdbe.monetdbe_prepare_cb", SQLSTATE(42000) "result table for prepared statement is wrong.");
    1134             : 
    1135           0 :         backend *be = NULL;
    1136           0 :         if ((mdbe->msg = getBackendContext(mdbe->c, &be)) != NULL)
    1137             :                 return mdbe->msg;
    1138             : 
    1139           0 :         if ( (mdbe->msg =  monetdbe_set_remote_results(be, tblname, results, nr_results)) != NULL)
    1140             :                 return mdbe->msg;
    1141             : 
    1142             :         BAT* btype = NULL;
    1143             :         BAT* bdigits = NULL;
    1144             :         BAT* bscale = NULL;
    1145             :         BAT* bschema = NULL;
    1146             :         BAT* btable = NULL;
    1147             :         BAT* bcolumn = NULL;
    1148             :         BAT* bimpl = NULL;
    1149             : 
    1150             :         size_t nparams = 0;
    1151           0 :         BATiter btype_iter = {0};
    1152           0 :         BATiter bcolumn_iter = {0};
    1153           0 :         BATiter btable_iter = {0};
    1154           0 :         BATiter bimpl_iter = {0};
    1155             :         char* function = NULL;
    1156             :         Symbol prg = NULL;
    1157             :         MalBlkPtr mb = NULL;
    1158             :         InstrPtr o = NULL, e = NULL, r = NULL;
    1159             :         sql_rel* rel = NULL;
    1160             :         list *args = NULL, *rets = NULL;
    1161             :         sql_allocator* sa = NULL;
    1162           0 :         ValRecord v = { .len=0 };
    1163           0 :         ptr vp = NULL;
    1164             :         struct callback_context* ccontext= NULL;
    1165             :         columnar_result_callback* rcb = NULL;
    1166             : 
    1167             :         str msg = MAL_SUCCEED;
    1168           0 :         if (!(btype             = BATdescriptor(results[0].id)) ||
    1169           0 :                 !(bdigits       = BATdescriptor(results[1].id)) ||
    1170           0 :                 !(bscale        = BATdescriptor(results[2].id)) ||
    1171           0 :                 !(bschema       = BATdescriptor(results[3].id)) ||
    1172           0 :                 !(btable        = BATdescriptor(results[4].id)) ||
    1173           0 :                 !(bcolumn       = BATdescriptor(results[5].id)) ||
    1174           0 :                 !(bimpl         = BATdescriptor(results[6].id)))
    1175             :         {
    1176           0 :                 msg = createException(SQL, "monetdbe.monetdbe_prepare_cb", SQLSTATE(HY005) "Cannot access column descriptor");
    1177           0 :                 goto cleanup;
    1178             :         }
    1179             : 
    1180           0 :         nparams = BATcount(btype);
    1181             : 
    1182           0 :         if (nparams     != BATcount(bdigits) ||
    1183           0 :                 nparams         != BATcount(bimpl) ||
    1184           0 :                 nparams         != BATcount(bscale) ||
    1185           0 :                 nparams         != BATcount(bschema) ||
    1186           0 :                 nparams         != BATcount(btable) ||
    1187           0 :                 nparams         != BATcount(bcolumn))
    1188             :         {
    1189           0 :                 msg = createException(SQL, "monetdbe.monetdbe_prepare_cb", SQLSTATE(42000) "Prepare results are incorrect");
    1190           0 :                 goto cleanup;
    1191             :         }
    1192             : 
    1193           0 :         btype_iter              = bat_iterator(btype);
    1194           0 :         bcolumn_iter            = bat_iterator(bcolumn);
    1195           0 :         btable_iter             = bat_iterator(btable);
    1196           0 :         bimpl_iter              = bat_iterator(bimpl);
    1197           0 :         function                =  BUNtvar(btable_iter, BATcount(btable)-1);
    1198             : 
    1199             :         {
    1200           0 :                 assert (((backend*)  mdbe->c->sqlcontext)->remote < INT_MAX);
    1201           0 :                 char nme[16]            = {0};
    1202           0 :                 const char* name        = number2name(nme, sizeof(nme), ++((backend*)  mdbe->c->sqlcontext)->remote);
    1203           0 :                 prg                                     = newFunction(userRef, putName(name), FUNCTIONsymbol);
    1204             :         }
    1205             : 
    1206           0 :         resizeMalBlk(prg->def, (int) nparams + 3 /*function declaration + remote.exec + return statement*/);
    1207           0 :         mb = prg->def;
    1208             : 
    1209           0 :         o = getInstrPtr(mb, 0);
    1210           0 :         o->retc = o->argc = 0;
    1211             : 
    1212           0 :         e = newInstructionArgs(mb, remoteRef, execRef, (int)(nparams + 5));
    1213           0 :         setDestVar(e, newTmpVariable(mb, TYPE_any));
    1214           0 :         e = pushStr(mb, e, mdbe->mid);
    1215           0 :         e = pushStr(mb, e, userRef);
    1216           0 :         e = pushStr(mb, e, function);
    1217             : 
    1218           0 :         rcb = GDKmalloc(sizeof(columnar_result_callback));
    1219           0 :         if (rcb == NULL) {
    1220           0 :                 msg = createException(MAL, "monetdbe.monetdbe_prepare_cb", MAL_MALLOC_FAIL);
    1221           0 :                 goto cleanup;
    1222             :         }
    1223             : 
    1224           0 :         ccontext = GDKzalloc(sizeof(struct callback_context));
    1225           0 :         if (ccontext == NULL) {
    1226           0 :                 msg = createException(MAL, "monetdbe.monetdbe_prepare_cb", MAL_MALLOC_FAIL);
    1227           0 :                 goto cleanup;
    1228             :         }
    1229             : 
    1230           0 :         ccontext->mdbe = mdbe;
    1231             : 
    1232           0 :         rcb->context = ccontext;
    1233           0 :         rcb->call = monetdbe_result_cb;
    1234             : 
    1235           0 :         vp = (ptr) rcb;
    1236             : 
    1237           0 :         VALset(&v, TYPE_ptr, &vp);
    1238           0 :         e = pushValue(mb, e, &v);
    1239             : 
    1240           0 :         r = newInstruction(mb, NULL, NULL);
    1241           0 :         r->barrier= RETURNsymbol;
    1242           0 :         r->argc= r->retc=0;
    1243             : 
    1244           0 :         sa = be->mvc->sa;
    1245             : 
    1246           0 :         args = new_exp_list(sa);
    1247           0 :         rets = new_exp_list(sa);
    1248             : 
    1249           0 :         for (size_t i = 0; i < nparams; i++) {
    1250             : 
    1251           0 :                 char* table     = BUNtvar(btable_iter, i);
    1252             : 
    1253           0 :                 sql_type *t = SA_ZNEW(sa, sql_type);
    1254           0 :                 char* name = BUNtvar(btype_iter, i);
    1255           0 :                 t->base.name = SA_STRDUP(sa, name);
    1256           0 :                 char* impl = BUNtvar(bimpl_iter, i);
    1257           0 :                 t->impl      = SA_STRDUP(sa, impl);
    1258           0 :                 t->localtype = ATOMindex(t->impl);
    1259             : 
    1260           0 :                 sql_subtype *st = SA_ZNEW(sa, sql_subtype);
    1261           0 :                 sql_init_subtype(st, t, (unsigned) *(int*) Tloc(bdigits, i), (unsigned) *(int*) Tloc(bscale, i));
    1262             : 
    1263           0 :                 if (strNil(table)) {
    1264             :                         // input argument
    1265             : 
    1266           0 :                         sql_arg *a = SA_ZNEW(sa, sql_arg);
    1267           0 :                         a->type = *st;
    1268           0 :                         append(args, a);
    1269             : 
    1270           0 :                         int idx = newVariable(mb, NULL, 0, t->localtype);
    1271           0 :                         o = pushArgument(mb, o, idx);
    1272             : 
    1273           0 :                         InstrPtr p = newFcnCall(mb, remoteRef, putRef);
    1274           0 :                         setArgType(mb, p, 0, TYPE_str);
    1275           0 :                         p = pushStr(mb, p, mdbe->mid);
    1276           0 :                         p = pushArgument(mb, p, idx);
    1277             : 
    1278           0 :                         e = pushArgument(mb, e, getArg(p, 0));
    1279             :                 }
    1280             :                 else {
    1281             :                         // output argument
    1282             : 
    1283           0 :                         char* column = BUNtvar(bcolumn_iter, i);
    1284           0 :                         sql_exp * c = exp_column(sa, table, column, st, CARD_MULTI, true, false);
    1285           0 :                         append(rets, c);
    1286             :                 }
    1287             :         }
    1288           0 :         pushInstruction(mb, e);
    1289           0 :         pushInstruction(mb, r);
    1290             : 
    1291           0 :         if ( (mdbe->msg = chkProgram(mdbe->c->usermodule, mb)) != MAL_SUCCEED ) {
    1292             :                 msg = mdbe->msg;
    1293           0 :                 goto cleanup;
    1294             :         }
    1295             : 
    1296           0 :         rel = rel_project(sa, NULL, rets);
    1297           0 :         be->q = qc_insert(be->mvc->qc, sa, rel, NULL, args, be->mvc->type, NULL, be->no_mitosis);
    1298           0 :         *prepare_id = be->q->id;
    1299             : 
    1300             :         /*
    1301             :          * HACK: we need to rename the Symbol aka MAL function to the query cache name.
    1302             :          * Basically we keep the MALblock but we destroy the containing old Symbol
    1303             :          * and create a new one with the correct name and set its MAL block pointer to
    1304             :          * point to the mal block we have created in this function.
    1305             :          */
    1306           0 :         prg->def = NULL;
    1307           0 :         freeSymbol(prg);
    1308           0 :         if ((prg = newFunction(userRef, putName(be->q->name), FUNCTIONsymbol)) == NULL) {
    1309           0 :                 msg = createException(MAL, "monetdbe.monetdbe_prepare_cb", MAL_MALLOC_FAIL);
    1310           0 :                 goto cleanup;
    1311             :         }
    1312           0 :         prg->def = mb;
    1313           0 :         setFunctionId(getSignature(prg), be->q->name);
    1314             : 
    1315             :         // finally add this beautiful new function to the local user module.
    1316           0 :         insertSymbol(mdbe->c->usermodule, prg);
    1317             : 
    1318           0 : cleanup:
    1319           0 :         if (bcolumn) {
    1320           0 :                 bat_iterator_end(&btype_iter);
    1321           0 :                 bat_iterator_end(&bcolumn_iter);
    1322           0 :                 bat_iterator_end(&btable_iter);
    1323           0 :                 bat_iterator_end(&bimpl_iter);
    1324             :         }
    1325             :         // clean these up
    1326           0 :         if (btype)              BBPunfix(btype->batCacheid);
    1327           0 :         if (bimpl)              BBPunfix(bimpl->batCacheid);
    1328           0 :         if (bdigits)    BBPunfix(bdigits->batCacheid);
    1329           0 :         if (bscale)             BBPunfix(bscale->batCacheid);
    1330           0 :         if (bschema)    BBPunfix(bschema->batCacheid);
    1331           0 :         if (btable)             BBPunfix(btable->batCacheid);
    1332           0 :         if (bcolumn)    BBPunfix(bcolumn->batCacheid);
    1333             : 
    1334           0 :         if (msg && rcb) GDKfree(rcb);
    1335           0 :         if (msg && ccontext) GDKfree(ccontext);
    1336             : 
    1337             :         return msg;
    1338             : }
    1339             : 
    1340             : static char*
    1341           1 : monetdbe_query_remote(monetdbe_database_internal *mdbe, char* query, monetdbe_result** result, monetdbe_cnt* affected_rows, int *prepare_id)
    1342             : {
    1343             :         const char *mod = "user";
    1344             :         char nme[16];
    1345             : 
    1346           1 :         Client c = mdbe->c;
    1347             : 
    1348           1 :         const char *name = number2name(nme, sizeof(nme), ++((backend*)  c->sqlcontext)->remote);
    1349           1 :         Symbol prg = newFunction(putName(mod), putName(name), FUNCTIONsymbol);
    1350             : 
    1351           1 :         if (prg == NULL) {
    1352           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_query_remote", MAL_MALLOC_FAIL));
    1353           0 :                 return mdbe->msg;
    1354             :         }
    1355             : 
    1356           1 :         MalBlkPtr mb = prg->def;
    1357             :         ValRecord v;
    1358             :         ptr vp;
    1359             :         columnar_result_callback* rcb;
    1360           1 :         InstrPtr f = getInstrPtr(mb, 0), r, p, e, o;
    1361             : 
    1362           1 :         f->retc = f->argc = 0;
    1363           1 :         o = newStmt(mb, remoteRef, putRef);
    1364           1 :         o = pushStr(mb, o, mdbe->mid);
    1365           1 :         o = pushBit(mb, o, TRUE);
    1366             : 
    1367           1 :         if (prepare_id) {
    1368             :                 size_t query_len, input_query_len, prep_len = 0;
    1369           0 :                 input_query_len = strlen(query);
    1370             :                 query_len = input_query_len + 3;
    1371           0 :                 const char PREPARE[] = "PREPARE ";
    1372             :                 prep_len = sizeof(PREPARE)-1;
    1373           0 :                 query_len += prep_len;
    1374             :                 char *nq = NULL;
    1375           0 :                 if (!(nq = GDKmalloc(query_len))) {
    1376           0 :                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_query_remote", "Could not setup query stream"));
    1377           0 :                         goto finalize;
    1378             :                 }
    1379           0 :                 strcpy(nq, PREPARE);
    1380           0 :                 strcpy(nq + prep_len, query);
    1381           0 :                 strcpy(nq + prep_len + input_query_len, "\n;");
    1382             :                 query = nq;
    1383             :         }
    1384             : 
    1385           1 :         p = newStmt(mb, remoteRef, putRef);
    1386           1 :         p = pushStr(mb, p, mdbe->mid);
    1387           1 :         p = pushStr(mb, p, query);
    1388             : 
    1389             : 
    1390           1 :         e = newInstruction(mb, remoteRef, execRef);
    1391           1 :         setDestVar(e, newTmpVariable(mb, TYPE_any));
    1392           1 :         e = pushStr(mb, e, mdbe->mid);
    1393           1 :         e = pushStr(mb, e, sqlRef);
    1394           1 :         e = pushStr(mb, e, evalRef);
    1395             : 
    1396             :         /*
    1397             :          * prepare the call back routine and its context
    1398             :          * and pass it over as a pointer to remote.exec.
    1399             :          */
    1400           1 :         rcb = GDKzalloc(sizeof(columnar_result_callback));
    1401           1 :         if (!rcb) {
    1402           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_query_remote", "Could not setup query stream"));
    1403           0 :                 goto finalize;
    1404             :         }
    1405             : 
    1406           1 :         if (!prepare_id) {
    1407             :                 struct callback_context* ccontext;
    1408           1 :                 ccontext                = GDKzalloc(sizeof(struct callback_context));
    1409           1 :                 ccontext->mdbe       = mdbe;
    1410           1 :                 rcb->context = ccontext;
    1411           1 :                 rcb->call            = monetdbe_result_cb;
    1412             :         }
    1413             :         else {
    1414             :                 struct prepare_callback_context* ccontext;
    1415           0 :                 ccontext                                = GDKzalloc(sizeof(struct prepare_callback_context));
    1416           0 :                 ccontext->mdbe                       = mdbe;
    1417           0 :                 ccontext->prepare_id = prepare_id;
    1418           0 :                 rcb->context                 = ccontext;
    1419           0 :                 rcb->call                            = monetdbe_prepare_cb;
    1420             :         }
    1421           1 :         if (!rcb->context) {
    1422           0 :                 GDKfree(rcb);
    1423           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_query_remote", "Could not setup query stream"));
    1424           0 :                 goto finalize;
    1425             :         }
    1426             : 
    1427           1 :         vp = (ptr) rcb;
    1428           1 :         VALset(&v, TYPE_ptr, &vp);
    1429           1 :         e = pushValue(mb, e, &v);
    1430             : 
    1431           1 :         e = pushArgument(mb, e, getArg(p, 0));
    1432           1 :         e = pushArgument(mb, e, getArg(o, 0));
    1433             : 
    1434           1 :         pushInstruction(mb, e);
    1435             : 
    1436           1 :         r = newInstruction(mb, NULL, NULL);
    1437           1 :         r->barrier= RETURNsymbol;
    1438           1 :         r->argc= r->retc=0;
    1439           1 :         pushInstruction(mb, r);
    1440             : 
    1441           1 :         if ( (mdbe->msg = chkProgram(c->usermodule, mb)) != MAL_SUCCEED )
    1442           0 :                 goto finalize;
    1443             : 
    1444           1 :         if ( (mdbe->msg = runMAL(c, mb, 0, NULL)) != MAL_SUCCEED )
    1445           0 :                 goto finalize;
    1446             : 
    1447           1 :         if (result) {
    1448           1 :                 if ((mdbe->msg = monetdbe_get_results(result, mdbe)) != MAL_SUCCEED)
    1449           0 :                         goto finalize;
    1450             : 
    1451           1 :                 mvc* m = NULL;
    1452           1 :                 backend * be = NULL;
    1453           1 :                 if ((mdbe->msg = getSQLContext(c, NULL, &m, &be)) != MAL_SUCCEED)
    1454           0 :                         goto finalize;
    1455             : 
    1456           1 :                 if (m->emode & m_prepare)
    1457           0 :                         ((monetdbe_result_internal*) *result)->type = Q_PREPARE;
    1458             :                 else
    1459           1 :                         ((monetdbe_result_internal*) *result)->type = (be->results) ? be->results->query_type : m->type;
    1460             : 
    1461             : 
    1462           1 :                 if (!be->results && be->rowcnt >= 0 && affected_rows)
    1463           0 :                         *affected_rows = be->rowcnt;
    1464             :         }
    1465             : 
    1466           0 : finalize:
    1467           1 :         freeSymbol(prg);
    1468           1 :         return mdbe->msg;
    1469             : }
    1470             : 
    1471             : char*
    1472           1 : monetdbe_query(monetdbe_database dbhdl, char* query, monetdbe_result** result, monetdbe_cnt* affected_rows)
    1473             : {
    1474           1 :         if (!dbhdl)
    1475             :                 return NULL;
    1476             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
    1477             : 
    1478           1 :         if (mdbe->mid) {
    1479           1 :                 mdbe->msg = monetdbe_query_remote(mdbe, query, result, affected_rows, NULL);
    1480             :         }
    1481             :         else {
    1482           0 :                 mdbe->msg = monetdbe_query_internal(mdbe, query, result, affected_rows, NULL, 'S');
    1483             :         }
    1484             : 
    1485           1 :         return mdbe->msg;
    1486             : }
    1487             : 
    1488             : char*
    1489           0 : monetdbe_prepare(monetdbe_database dbhdl, char* query, monetdbe_statement **stmt, monetdbe_result** result)
    1490             : {
    1491           0 :         if (!dbhdl)
    1492             :                 return NULL;
    1493             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
    1494             : 
    1495           0 :         int prepare_id = 0;
    1496             : 
    1497           0 :         if (!stmt) {
    1498           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_prepare", "Parameter stmt is NULL"));
    1499           0 :                 assert(mdbe->msg != MAL_SUCCEED); /* help Coverity */
    1500           0 :         } else if (mdbe->mid) {
    1501           0 :                 mdbe->msg = monetdbe_query_remote(mdbe, query, result, NULL, &prepare_id);
    1502             :         } else {
    1503           0 :                 *stmt = NULL;
    1504           0 :                 mdbe->msg = monetdbe_query_internal(mdbe, query, result, NULL, &prepare_id, 'S');
    1505             :         }
    1506             : 
    1507           0 :         if (mdbe->msg == MAL_SUCCEED) {
    1508           0 :                 mvc *m = ((backend *) mdbe->c->sqlcontext)->mvc;
    1509           0 :                 monetdbe_stmt_internal *stmt_internal = (monetdbe_stmt_internal*)GDKzalloc(sizeof(monetdbe_stmt_internal));
    1510           0 :                 cq *q = qc_find(m->qc, prepare_id);
    1511             : 
    1512           0 :                 if (q && stmt_internal) {
    1513           0 :                         Symbol s = findSymbolInModule(mdbe->c->usermodule, q->f->imp);
    1514           0 :                         InstrPtr p = s->def->stmt[0];
    1515           0 :                         stmt_internal->mdbe = mdbe;
    1516           0 :                         stmt_internal->q = q;
    1517           0 :                         stmt_internal->retc = p->retc;
    1518           0 :                         stmt_internal->res.nparam = list_length(q->f->ops);
    1519           0 :                         stmt_internal->args = (ValPtr*)GDKmalloc(sizeof(ValPtr) * (stmt_internal->res.nparam + stmt_internal->retc));
    1520           0 :                         stmt_internal->data = (ValRecord*)GDKzalloc(sizeof(ValRecord) * (stmt_internal->res.nparam+1));
    1521           0 :                         stmt_internal->res.type = (monetdbe_types*)GDKmalloc(sizeof(monetdbe_types) * (stmt_internal->res.nparam+1));
    1522           0 :                         if (!stmt_internal->res.type || !stmt_internal->data || !stmt_internal->args) {
    1523           0 :                                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_prepare", MAL_MALLOC_FAIL));
    1524           0 :                         } else if (q->f->ops) {
    1525             :                                 int i = 0;
    1526           0 :                                 for (node *n = q->f->ops->h; n; n = n->next, i++) {
    1527           0 :                                         sql_arg *a = n->data;
    1528             :                                         sql_subtype *t = &a->type;
    1529           0 :                                         stmt_internal->res.type[i] = embedded_type(t->type->localtype);
    1530           0 :                                         stmt_internal->args[i+stmt_internal->retc] = &stmt_internal->data[i];
    1531             :                                 }
    1532             :                         }
    1533           0 :                 } else if (!stmt_internal)
    1534           0 :                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_prepare", MAL_MALLOC_FAIL));
    1535             : 
    1536           0 :                 if (mdbe->msg == MAL_SUCCEED)
    1537           0 :                         *stmt = (monetdbe_statement*)stmt_internal;
    1538           0 :                 else if (stmt_internal) {
    1539           0 :                         GDKfree(stmt_internal->data);
    1540           0 :                         GDKfree(stmt_internal->args);
    1541           0 :                         GDKfree(stmt_internal->res.type);
    1542           0 :                         GDKfree(stmt_internal);
    1543           0 :                         *stmt = NULL;
    1544             :                 }
    1545             :         }
    1546             : 
    1547           0 :         return mdbe->msg;
    1548             : }
    1549             : 
    1550             : char*
    1551           0 : monetdbe_bind(monetdbe_statement *stmt, void *data, size_t i)
    1552             : {
    1553             :         monetdbe_stmt_internal *stmt_internal = (monetdbe_stmt_internal*)stmt;
    1554             : 
    1555           0 :         if (i >= stmt->nparam)
    1556           0 :                 return createException(MAL, "monetdbe.monetdbe_bind", "Parameter %zu not bound to a value", i);
    1557           0 :         sql_arg *a = (sql_arg*)list_fetch(stmt_internal->q->f->ops, (int) i);
    1558           0 :         assert(a);
    1559           0 :         int tpe = a->type.type->localtype;
    1560           0 :         stmt_internal->data[i].vtype = tpe;
    1561             : 
    1562           0 :         const void* nil = (tpe>=0)?ATOMnilptr(tpe):NULL;
    1563           0 :         if (!data) {
    1564           0 :                 VALset(&stmt_internal->data[i], tpe, (ptr)nil);
    1565           0 :         } else if (tpe == TYPE_timestamp) {
    1566             :                 monetdbe_data_timestamp* ts = (monetdbe_data_timestamp*)data;
    1567           0 :                 timestamp t = *(timestamp*) nil;
    1568           0 :                 if(!timestamp_is_null(ts))
    1569           0 :                         t = timestamp_from_data(ts);
    1570           0 :                 VALset(&stmt_internal->data[i], tpe, &t);
    1571           0 :         } else if (tpe == TYPE_date) {
    1572             :                 monetdbe_data_date* de = (monetdbe_data_date*)data;
    1573           0 :                 date d = *(date*) nil;
    1574           0 :                 if(!date_is_null(de))
    1575           0 :                         d = date_from_data(de);
    1576           0 :                 VALset(&stmt_internal->data[i], tpe, &d);
    1577           0 :         } else if (tpe == TYPE_daytime) {
    1578             :                 monetdbe_data_time* t = (monetdbe_data_time*)data;
    1579           0 :                 daytime dt = *(daytime*) nil;
    1580             : 
    1581           0 :                 if(!time_is_null(t))
    1582           0 :                         dt = time_from_data(t);
    1583           0 :                 VALset(&stmt_internal->data[i], tpe, &dt);
    1584           0 :         } else if (tpe == TYPE_blob) {
    1585             :                 monetdbe_data_blob *be = (monetdbe_data_blob*)data;
    1586             :                 blob *b = (blob*)nil;
    1587             :                 if (!blob_is_null(be)) {
    1588           0 :                         size_t len = be->size;
    1589           0 :                         b = (blob*) GDKmalloc(blobsize(len));
    1590           0 :                         if (b == NULL) {
    1591           0 :                                 set_error(stmt_internal->mdbe, createException(MAL, "monetdbe.monetdbe_bind", MAL_MALLOC_FAIL));
    1592           0 :                                 return stmt_internal->mdbe->msg;
    1593             :                         }
    1594           0 :                         b->nitems = len;
    1595           0 :                         memcpy(b->data, be->data, len);
    1596             :                 }
    1597           0 :                 VALset(&stmt_internal->data[i], tpe, b);
    1598           0 :         } else if (tpe == TYPE_str) {
    1599           0 :                 char *val = GDKstrdup(data);
    1600             : 
    1601           0 :                 if (val == NULL) {
    1602           0 :                         set_error(stmt_internal->mdbe, createException(MAL, "monetdbe.monetdbe_bind", MAL_MALLOC_FAIL));
    1603           0 :                         return stmt_internal->mdbe->msg;
    1604             :                 }
    1605           0 :                 VALset(&stmt_internal->data[i], tpe, val);
    1606             :         } else {
    1607           0 :                 VALset(&stmt_internal->data[i], tpe, data);
    1608             :         }
    1609             :         return MAL_SUCCEED;
    1610             : }
    1611             : 
    1612             : char*
    1613           0 : monetdbe_execute(monetdbe_statement *stmt, monetdbe_result **result, monetdbe_cnt *affected_rows)
    1614             : {
    1615             :         monetdbe_result_internal *res_internal = NULL;
    1616             :         monetdbe_stmt_internal *stmt_internal = (monetdbe_stmt_internal*)stmt;
    1617           0 :         backend *b = (backend *) stmt_internal->mdbe->c->sqlcontext;
    1618           0 :         mvc *m = b->mvc;
    1619             :         monetdbe_database_internal *mdbe = stmt_internal->mdbe;
    1620           0 :         MalStkPtr glb = NULL;
    1621           0 :         cq *q = stmt_internal->q;
    1622             :         Symbol s = NULL;
    1623             : 
    1624           0 :         if ((mdbe->msg = SQLtrans(m)) != MAL_SUCCEED)
    1625             :                 return mdbe->msg;
    1626             : 
    1627             :         /* check if all inputs are bound */
    1628           0 :         for(int i = 0; i< list_length(stmt_internal->q->f->ops); i++){
    1629           0 :                 if (!stmt_internal->data[i].vtype) {
    1630           0 :                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_execute", "Parameter %d not bound to a value", i));
    1631           0 :                         goto cleanup;
    1632             :                 }
    1633             :         }
    1634             : 
    1635           0 :         s = findSymbolInModule(mdbe->c->usermodule, q->f->imp);
    1636           0 :         if ((mdbe->msg = callMAL(mdbe->c, s->def, &glb, stmt_internal->args, 0)) != MAL_SUCCEED)
    1637           0 :                 goto cleanup;
    1638             : 
    1639           0 :         if (b->rowcnt >= 0 && affected_rows)
    1640           0 :                 *affected_rows = b->rowcnt;
    1641             : 
    1642           0 :         if (result) {
    1643           0 :                 if ((mdbe->msg = monetdbe_get_results(result, mdbe)) != MAL_SUCCEED) {
    1644           0 :                         goto cleanup;
    1645             :                 }
    1646             : 
    1647           0 :                 (*(monetdbe_result_internal**) result)->type = (b->results) ? Q_TABLE : Q_UPDATE;
    1648             :                 res_internal = *(monetdbe_result_internal**)result;
    1649             :         }
    1650             : 
    1651           0 : cleanup:
    1652           0 :         GDKfree(glb);
    1653           0 :         return commit_action(m, stmt_internal->mdbe, result, res_internal);
    1654             : }
    1655             : 
    1656             : char*
    1657           0 : monetdbe_cleanup_statement(monetdbe_database dbhdl, monetdbe_statement *stmt)
    1658             : {
    1659             :         monetdbe_stmt_internal *stmt_internal = (monetdbe_stmt_internal*)stmt;
    1660             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
    1661           0 :         mvc *m = ((backend *) mdbe->c->sqlcontext)->mvc;
    1662           0 :         cq *q = stmt_internal->q;
    1663             : 
    1664           0 :         assert(!stmt_internal->mdbe || mdbe == stmt_internal->mdbe);
    1665             : 
    1666           0 :         for (size_t i = 0; i < stmt_internal->res.nparam + 1; i++) {
    1667           0 :                 ValPtr data = &stmt_internal->data[i];
    1668           0 :                 if (data->vtype == TYPE_str || data->vtype == TYPE_blob) {
    1669           0 :                         GDKfree(data->val.pval);
    1670             :                 }
    1671             :         }
    1672             : 
    1673           0 :         GDKfree(stmt_internal->data);
    1674           0 :         GDKfree(stmt_internal->args);
    1675           0 :         GDKfree(stmt_internal->res.type);
    1676           0 :         GDKfree(stmt_internal);
    1677             : 
    1678           0 :         if (q)
    1679           0 :                 qc_delete(m->qc, q);
    1680           0 :         return MAL_SUCCEED;
    1681             : }
    1682             : 
    1683             : char*
    1684           1 : monetdbe_cleanup_result(monetdbe_database dbhdl, monetdbe_result* result)
    1685             : {
    1686             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
    1687             :         monetdbe_result_internal* res = (monetdbe_result_internal *) result;
    1688             : 
    1689             : 
    1690           1 :         if (!result) {
    1691           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_cleanup_result_internal", "Parameter result is NULL"));
    1692             :         } else {
    1693           1 :                 mdbe->msg = monetdbe_cleanup_result_internal(mdbe, res);
    1694             :         }
    1695             : 
    1696           1 :         return mdbe->msg;
    1697             : }
    1698             : 
    1699             : static inline void
    1700           0 : cleanup_get_columns_result(size_t column_count, char ** column_names, int *column_types)
    1701             : {
    1702           0 :                 if (column_names) for (size_t c = 0; c < column_count; c++) GDKfree(column_names[c]);
    1703             : 
    1704           0 :                 GDKfree(column_names);
    1705           0 :                 GDKfree(column_types);
    1706             : 
    1707             :                 column_names = NULL;
    1708             :                 column_types = NULL;
    1709           0 : }
    1710             : 
    1711             : static char *
    1712           0 : escape_identifier(const char *s) /* Escapes a SQL identifier string, ie the " and \ characters */
    1713             : {
    1714             :         char *ret = NULL, *q;
    1715             :         const char *p = s;
    1716             : 
    1717             :         /* At most we will need 2*strlen(s) + 1 characters */
    1718           0 :         if (!(ret = (char *)GDKmalloc(2*strlen(s) + 1)))
    1719             :                 return NULL;
    1720             : 
    1721           0 :         for (q = ret; *p; p++, q++) {
    1722           0 :                 *q = *p;
    1723           0 :                 if (*p == '"')
    1724           0 :                         *(++q) = '"';
    1725           0 :                 else if (*p == '\\')
    1726           0 :                         *(++q) = '\\';
    1727             :         }
    1728             : 
    1729           0 :         *q = '\0';
    1730           0 :         return ret;
    1731             : }
    1732             : 
    1733             : static char*
    1734           0 : monetdbe_get_columns_remote(monetdbe_database_internal *mdbe, const char* schema_name, const char *table_name, size_t *column_count,
    1735             :                                         char ***column_names, int **column_types)
    1736             : {
    1737             :         char buf[1024], *escaped_schema_name = NULL, *escaped_table_name = NULL;
    1738             : 
    1739           0 :         if (schema_name && !(escaped_schema_name = escape_identifier(schema_name))) {
    1740           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_columns", MAL_MALLOC_FAIL));
    1741           0 :                 return mdbe->msg;
    1742             :         }
    1743           0 :         if (!(escaped_table_name = escape_identifier(table_name))) {
    1744           0 :                 GDKfree(escaped_schema_name);
    1745           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_columns", MAL_MALLOC_FAIL));
    1746           0 :                 return mdbe->msg;
    1747             :         }
    1748             : 
    1749           0 :         int len = snprintf(buf, 1024, "SELECT * FROM %s%s%s\"%s\" WHERE FALSE;",
    1750             :                                            escaped_schema_name ? "\"" : "",  escaped_schema_name ? escaped_schema_name : "",
    1751             :                                            escaped_schema_name ? escaped_schema_name : "\".", escaped_table_name);
    1752           0 :         GDKfree(escaped_schema_name);
    1753           0 :         GDKfree(escaped_table_name);
    1754           0 :         if (len == -1 || len >= 1024) {
    1755           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_columns", "Schema and table path is too large"));
    1756           0 :                 return mdbe->msg;
    1757             :         }
    1758             : 
    1759           0 :         monetdbe_result* result = NULL;
    1760             : 
    1761           0 :         if ((mdbe->msg = monetdbe_query_remote(mdbe, buf, &result, NULL, NULL)) != MAL_SUCCEED) {
    1762             :                 return mdbe->msg;
    1763             :         }
    1764             : 
    1765           0 :         *column_count = result->ncols;
    1766           0 :         *column_names = GDKzalloc(sizeof(char*) * result->ncols);
    1767           0 :         *column_types = GDKzalloc(sizeof(int) * result->ncols);
    1768             : 
    1769             : 
    1770           0 :         if (*column_names == NULL || *column_types == NULL)
    1771           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_columns", MAL_MALLOC_FAIL));
    1772             : 
    1773           0 :         if (!mdbe->msg)
    1774           0 :                 for (size_t c = 0; c < result->ncols; c++) {
    1775             :                         monetdbe_column* rcol;
    1776           0 :                         if ((mdbe->msg = monetdbe_result_fetch(result, &rcol, c)) != NULL) {
    1777             :                                 break;
    1778             :                         }
    1779             : 
    1780           0 :                         if (((*column_names)[c] = GDKstrdup(rcol->name)) == NULL) {
    1781           0 :                                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_columns", MAL_MALLOC_FAIL));
    1782           0 :                                 break;
    1783             :                         }
    1784           0 :                         (*column_types)[c] = rcol->type;
    1785             :                 }
    1786             : 
    1787             :         // cleanup
    1788           0 :         if  (result) {
    1789           0 :                 char* msg = monetdbe_cleanup_result_internal(mdbe, (monetdbe_result_internal*) result);
    1790             : 
    1791           0 :                 if (msg && mdbe->msg) {
    1792           0 :                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_columns", "multiple errors: %s; %s", mdbe->msg, msg));
    1793             :                 }
    1794           0 :                 else if (msg) {
    1795           0 :                         mdbe->msg = msg;
    1796             :                 }
    1797             :         }
    1798             : 
    1799           0 :         if (mdbe->msg ) {
    1800           0 :                 cleanup_get_columns_result(*column_count, *column_names, *column_types);
    1801             :         }
    1802             : 
    1803           0 :         return mdbe->msg;
    1804             : }
    1805             : 
    1806             : char*
    1807           0 : monetdbe_get_columns(monetdbe_database dbhdl, const char *schema_name, const char *table_name, size_t *column_count,
    1808             :                                         char ***column_names, int **column_types)
    1809             : {
    1810             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
    1811           0 :         mvc *m = NULL;
    1812             :         sql_table *t = NULL;
    1813             :         int columns = 0;
    1814             : 
    1815           0 :         if ((mdbe->msg = validate_database_handle(mdbe, "monetdbe.monetdbe_get_columns")) != MAL_SUCCEED) {
    1816             :                 return mdbe->msg;
    1817             :         }
    1818           0 :         if (!column_count) {
    1819           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_columns", "Parameter column_count is NULL"));
    1820           0 :                 return mdbe->msg;
    1821             :         }
    1822           0 :         if (!column_names) {
    1823           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_columns", "Parameter column_names is NULL"));
    1824           0 :                 return mdbe->msg;
    1825             :         }
    1826           0 :         if (!column_types) {
    1827           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_columns", "Parameter column_types is NULL"));
    1828           0 :                 return mdbe->msg;
    1829             :         }
    1830           0 :         if (!table_name) {
    1831           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_columns", "Parameter table_name is NULL"));
    1832           0 :                 return mdbe->msg;
    1833             :         }
    1834             : 
    1835           0 :         if (mdbe->mid) {
    1836           0 :                 return monetdbe_get_columns_remote(mdbe, schema_name, table_name, column_count, column_names, column_types);
    1837             :         }
    1838             : 
    1839           0 :         if ((mdbe->msg = getSQLContext(mdbe->c, NULL, &m, NULL)) != MAL_SUCCEED)
    1840             :                 return mdbe->msg;
    1841           0 :         if ((mdbe->msg = SQLtrans(m)) != MAL_SUCCEED)
    1842             :                 return mdbe->msg;
    1843           0 :         if (!(t = find_table_or_view_on_scope(m, NULL, schema_name, table_name, "CATALOG", false))) {
    1844           0 :                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_get_columns", "%s", m->errstr + 6)); /* Skip error code */
    1845           0 :                 goto cleanup;
    1846             :         }
    1847             : 
    1848           0 :         columns = ol_length(t->columns);
    1849           0 :         *column_count = columns;
    1850           0 :         *column_names = GDKzalloc(sizeof(char*) * columns);
    1851           0 :         *column_types = GDKzalloc(sizeof(int) * columns);
    1852           0 :         if (*column_names == NULL || *column_types == NULL) {
    1853           0 :                 if (*column_names) {
    1854           0 :                         GDKfree(*column_names);
    1855           0 :                         *column_names = NULL;
    1856             :                 }
    1857           0 :                 if (*column_types) {
    1858           0 :                         GDKfree(*column_types);
    1859           0 :                         *column_types = NULL;
    1860             :                 }
    1861           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_get_columns", MAL_MALLOC_FAIL));
    1862           0 :                 goto cleanup;
    1863             :         }
    1864             : 
    1865           0 :         for (node *n = ol_first_node(t->columns); n; n = n->next) {
    1866           0 :                 sql_column *col = n->data;
    1867           0 :                 (*column_names)[col->colnr] = col->base.name;
    1868           0 :                 (*column_types)[col->colnr] = embedded_type(col->type.type->localtype);
    1869             :         }
    1870             : 
    1871           0 : cleanup:
    1872           0 :         mdbe->msg = commit_action(m, mdbe, NULL, NULL);
    1873             : 
    1874           0 :         return mdbe->msg;
    1875             : }
    1876             : 
    1877             : #define GENERATE_BASE_HEADERS(type, tpename) \
    1878             :         static int tpename##_is_null(type *value)
    1879             : 
    1880             : #define GENERATE_BASE_FUNCTIONS(tpe, tpename, mname) \
    1881             :         GENERATE_BASE_HEADERS(tpe, tpename); \
    1882             :         static int tpename##_is_null(tpe *value) { return *value == mname##_nil; }
    1883             : 
    1884             : #ifdef bool
    1885             : #undef bool
    1886             : #endif
    1887             : 
    1888           0 : GENERATE_BASE_FUNCTIONS(int8_t, bool, bit)
    1889           0 : GENERATE_BASE_FUNCTIONS(int8_t, int8_t, bte)
    1890           0 : GENERATE_BASE_FUNCTIONS(int16_t, int16_t, sht)
    1891           0 : GENERATE_BASE_FUNCTIONS(int32_t, int32_t, int)
    1892           0 : GENERATE_BASE_FUNCTIONS(int64_t, int64_t, lng)
    1893             : #ifdef HAVE_HGE
    1894           0 : GENERATE_BASE_FUNCTIONS(__int128, int128_t, hge)
    1895             : #endif
    1896           0 : GENERATE_BASE_FUNCTIONS(size_t, size_t, oid)
    1897             : 
    1898           0 : GENERATE_BASE_FUNCTIONS(float, float, flt)
    1899           0 : GENERATE_BASE_FUNCTIONS(double, double, dbl)
    1900             : 
    1901             : GENERATE_BASE_HEADERS(char*, str);
    1902             : GENERATE_BASE_HEADERS(monetdbe_data_blob, blob);
    1903             : 
    1904             : GENERATE_BASE_HEADERS(monetdbe_data_date, date);
    1905             : GENERATE_BASE_HEADERS(monetdbe_data_time, time);
    1906             : GENERATE_BASE_HEADERS(monetdbe_data_timestamp, timestamp);
    1907             : 
    1908             : #define GENERATE_BAT_INPUT_BASE(tpe)                                                                    \
    1909             :         monetdbe_column_##tpe *bat_data = GDKzalloc(sizeof(monetdbe_column_##tpe));     \
    1910             :         if (!bat_data) {                                                                                                        \
    1911             :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_result_fetch", MAL_MALLOC_FAIL)); \
    1912             :                 goto cleanup;                                                                                                   \
    1913             :         }                                                                                                                                       \
    1914             :         bat_data->type = monetdbe_##tpe;                                                                     \
    1915             :         bat_data->is_null = tpe##_is_null;                                                                   \
    1916             :         if (sqltpe->type->radix == 10) bat_data->scale = pow(10, sqltpe->scale); \
    1917             :         column_result = (monetdbe_column*) bat_data;
    1918             : 
    1919             : #define GENERATE_BAT_INPUT(b, tpe, tpe_name, mtype)                                             \
    1920             :         {                                                                                                                                       \
    1921             :                 GENERATE_BAT_INPUT_BASE(tpe_name);                                                              \
    1922             :                 bat_data->count = (size_t) mres->nrows;                                                   \
    1923             :                 bat_data->null_value = mtype##_nil;                                                          \
    1924             :                 if (bat_data->count) {                                                                                       \
    1925             :                         bat_data->data = GDKzalloc(bat_data->count * sizeof(bat_data->null_value)); \
    1926             :                         if (!bat_data->data) {                                                                               \
    1927             :                                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_result_fetch", MAL_MALLOC_FAIL)); \
    1928             :                                 goto cleanup;                                                                                   \
    1929             :                         }                                                                                                                       \
    1930             :                 }                                                                                                                               \
    1931             :                 size_t it = 0;                                                                                                  \
    1932             :                 mtype* val = (mtype*)Tloc(b, 0);                                                                \
    1933             :                 /* bat is dense, materialize it */                                                              \
    1934             :                 for (it = 0; it < bat_data->count; it++, val++)                                   \
    1935             :                         bat_data->data[it] = (tpe) *val;                                                     \
    1936             :         }
    1937             : 
    1938             : static char*
    1939           0 : append_create_remote_append_mal_program(
    1940             :         Symbol* prg,
    1941             :         sql_schema **s,
    1942             :         sql_table **t,
    1943             :         Client c, const char* schema, const char* table, size_t ccount, char** cnames, int* ctypes) {
    1944             : 
    1945             :         char* msg                                       = MAL_SUCCEED;
    1946           0 :         char buf[16]                            = {0};
    1947           0 :         char* remote_program_name       = number2name(buf, sizeof(buf), ++((backend*) c->sqlcontext)->remote);
    1948             : 
    1949           0 :         assert(s && t);
    1950           0 :         assert(c->sqlcontext && ((backend *) c->sqlcontext)->mvc);
    1951             :         mvc* m = ((backend *) c->sqlcontext)->mvc;
    1952             : 
    1953             :         Symbol _prg;
    1954             :         MalBlkPtr mb = NULL;
    1955             :         InstrPtr f = NULL, v = NULL, a = NULL, r = NULL;
    1956             :         int mvc_id = -1;
    1957             : 
    1958           0 :         if (!(*s = mvc_bind_schema(m, "tmp"))) {
    1959           0 :                 return createException(MAL, "monetdbe.monetdbe_append", MAL_MALLOC_FAIL);
    1960             :         }
    1961             : 
    1962           0 :         if (!(*t = sql_trans_create_table(m->session->tr, *s, table, NULL, tt_table, false, SQL_DECLARED_TABLE, CA_COMMIT, -1, 0))) {
    1963           0 :                 return createException(SQL, "monetdbe.monetdbe_append", "Cannot create temporary table");
    1964             :         }
    1965             : 
    1966           0 :         assert(prg);
    1967             : 
    1968           0 :         *prg    = NULL;
    1969           0 :         _prg    = newFunction(userRef, putName(remote_program_name), FUNCTIONsymbol); // remote program
    1970           0 :         mb              = _prg->def;
    1971             : 
    1972             :         { // START OF HACK
    1973             :                 /*
    1974             :                  * This is a hack to make sure that the serialized remote program is correctly parsed on the remote side.
    1975             :                  * Since the mal serializer (mal_listing) on the local side will use generated variable names,
    1976             :                  * The parsing process on the remote side can and will clash with generated variable names on the remote side.
    1977             :                  * Because serialiser and the parser will both use the same namespace of generated variable names.
    1978             :                  * Adding an offset to the counter that generates the variable names on the local side
    1979             :                  * circumvents this shortcoming in the MAL parser.
    1980             :                  */
    1981             : 
    1982           0 :                 assert(mb->vid == 0);
    1983             : 
    1984             :                 /*
    1985             :                         * Comments generate variable names during parsing:
    1986             :                         * sql.mvc has one comment and for each column there is one sql.append statement plus comment.
    1987             :                         */
    1988           0 :                 const int nr_of_comments = (int) (1 + ccount);
    1989             :                 /*
    1990             :                         * constant terms generate variable names during parsing:
    1991             :                         * Each sql.append has three constant terms: schema + table + column_name.
    1992             :                         * There is one sql.append stmt for each column.
    1993             :                         */
    1994           0 :                 const int nr_of_constant_terms =  (int)  (3 * ccount);
    1995           0 :                 mb->vid = nr_of_comments + nr_of_constant_terms;
    1996             :         } // END OF HACK
    1997             : 
    1998           0 :         f = getInstrPtr(mb, 0);
    1999           0 :         f->retc = f->argc = 0;
    2000           0 :         f = pushReturn(mb, f, newTmpVariable(mb, TYPE_int));
    2001           0 :         v = newFcnCall(mb, sqlRef, mvcRef);
    2002           0 :         setArgType(mb, v, 0, TYPE_int);
    2003             : 
    2004           0 :         mvc_id = getArg(v, 0);
    2005             : 
    2006           0 :         sqlstore *store = m->session->tr->store;
    2007           0 :         for (size_t i = 0; i < ccount; i++) {
    2008             : 
    2009           0 :                 sql_type *tpe = SA_ZNEW(m->sa, sql_type);
    2010           0 :                 tpe->localtype = monetdbe_type((monetdbe_types) ctypes[i]);
    2011           0 :                 sql_subtype *st = SA_ZNEW(m->sa, sql_subtype);
    2012           0 :                 sql_init_subtype(st, tpe, 0, 0);
    2013             : 
    2014             :                 sql_column* col;
    2015           0 :                 if (!(col = mvc_create_column(m, *t, cnames[i], st))) {
    2016           0 :                         msg = createException(MAL, "monetdbe.monetdbe_append", MAL_MALLOC_FAIL);
    2017           0 :                         goto cleanup;
    2018             :                 }
    2019             : 
    2020           0 :                 if (store->storage_api.create_col(m->session->tr, col) != LOG_OK) {
    2021           0 :                         msg = createException(MAL, "monetdbe.monetdbe_append", MAL_MALLOC_FAIL);
    2022           0 :                         goto cleanup;
    2023             :                 }
    2024             : 
    2025           0 :                 int idx = newTmpVariable(mb, newBatType(tpe->localtype));
    2026           0 :                 f = pushArgument(mb, f, idx);
    2027             : 
    2028           0 :                 a = newFcnCall(mb, sqlRef, appendRef);
    2029           0 :                 setArgType(mb, a, 0, TYPE_int);
    2030           0 :                 a = pushArgument(mb, a, mvc_id);
    2031           0 :                 a = pushStr(mb, a, schema ? schema : "sys"); /* TODO this should be better */
    2032           0 :                 a = pushStr(mb, a, table);
    2033           0 :                 a = pushStr(mb, a, cnames[i]);
    2034           0 :                 a = pushArgument(mb, a, idx);
    2035             : 
    2036           0 :                 mvc_id = getArg(a, 0);
    2037             :         }
    2038             : 
    2039           0 :         r = newInstruction(mb, NULL, NULL);
    2040           0 :         r->barrier= RETURNsymbol;
    2041           0 :         r->retc = r->argc = 0;
    2042           0 :         r = pushReturn(mb, r, mvc_id);
    2043           0 :         r = pushArgument(mb, r, mvc_id);
    2044           0 :         pushInstruction(mb, r);
    2045             : 
    2046           0 :         pushEndInstruction(mb);
    2047             : 
    2048           0 :         if ( (msg = chkProgram(c->usermodule, mb)) != MAL_SUCCEED ) {
    2049           0 :                 goto cleanup;
    2050             :         }
    2051             : 
    2052             :         assert(msg == MAL_SUCCEED);
    2053           0 :         *prg = _prg;
    2054           0 :         return msg;
    2055             : 
    2056           0 : cleanup:
    2057           0 :         assert(msg != MAL_SUCCEED);
    2058           0 :         freeSymbol(_prg);
    2059           0 :         *prg = NULL;
    2060           0 :         return msg;
    2061             : }
    2062             : 
    2063             : char*
    2064           0 : monetdbe_append(monetdbe_database dbhdl, const char *schema, const char *table, monetdbe_column **input, size_t column_count)
    2065             : {
    2066             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
    2067           0 :         mvc *m = NULL;
    2068           0 :         sql_table *t = NULL;
    2069             :         size_t i, cnt;
    2070             :         node *n;
    2071           0 :         Symbol remote_prg = NULL;
    2072           0 :         BAT *pos = NULL;
    2073             :         BUN offset;
    2074             : 
    2075           0 :         if ((mdbe->msg = validate_database_handle(mdbe, "monetdbe.monetdbe_append")) != MAL_SUCCEED) {
    2076             :                 return mdbe->msg;
    2077             :         }
    2078             : 
    2079           0 :         if ((mdbe->msg = getSQLContext(mdbe->c, NULL, &m, NULL)) != MAL_SUCCEED) {
    2080           0 :                 mdbe->msg = commit_action(m, mdbe, NULL, NULL);
    2081           0 :                 return mdbe->msg;
    2082             :         }
    2083           0 :         sqlstore *store = m->session->tr->store;
    2084             : 
    2085           0 :         if (table == NULL) {
    2086           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_append", "table parameter is NULL"));
    2087           0 :                 goto cleanup;
    2088             :         }
    2089           0 :         if (input == NULL) {
    2090           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_append", "input parameter is NULL"));
    2091           0 :                 goto cleanup;
    2092             :         }
    2093           0 :         if (column_count < 1) {
    2094           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_append", "column_count must be higher than 0"));
    2095           0 :                 goto cleanup;
    2096             :         }
    2097             : 
    2098           0 :         if (mdbe->mid) {
    2099             :                 // We are going to insert the data into a temporary table which is used in the coming remote logic.
    2100             : 
    2101           0 :                 size_t actual_column_count = 0;
    2102           0 :                 char** actual_column_names = NULL;
    2103           0 :                 int* actual_column_types = NULL;
    2104           0 :                 sql_schema* s = NULL;
    2105             : 
    2106           0 :                 if ((mdbe->msg = monetdbe_get_columns_remote(
    2107             :                                 mdbe,
    2108             :                                 schema,
    2109             :                                 table,
    2110             :                                 &actual_column_count,
    2111             :                                 &actual_column_names,
    2112             :                                 &actual_column_types)) != MAL_SUCCEED) {
    2113           0 :                         goto remote_cleanup;
    2114             :                 }
    2115             : 
    2116           0 :                 if ((mdbe->msg = SQLtrans(m)) != MAL_SUCCEED) {
    2117           0 :                         goto remote_cleanup;
    2118             :                 }
    2119             : 
    2120           0 :                 if ((mdbe->msg = append_create_remote_append_mal_program
    2121             :                                                         (&remote_prg,
    2122             :                                                         &s,
    2123             :                                                         &t,
    2124             :                                                         mdbe->c,
    2125             :                                                         schema,
    2126             :                                                         table,
    2127             :                                                         actual_column_count,
    2128             :                                                         actual_column_names,
    2129             :                                                         actual_column_types)) != MAL_SUCCEED) {
    2130           0 :                         goto remote_cleanup;
    2131             :                 }
    2132             : 
    2133           0 :                 insertSymbol(mdbe->c->usermodule, remote_prg);
    2134             : 
    2135           0 : remote_cleanup:
    2136           0 :                 if (mdbe->msg) {
    2137           0 :                         cleanup_get_columns_result(actual_column_count, actual_column_names, actual_column_types);
    2138           0 :                         freeSymbol(remote_prg);
    2139           0 :                         goto cleanup;
    2140             :                 }
    2141             :         } else {
    2142             :                 // !mdbe->mid
    2143             :                 // inserting into existing local table.
    2144             :                 sql_part *pt = NULL;
    2145             : 
    2146           0 :                 if ((mdbe->msg = SQLtrans(m)) != MAL_SUCCEED)
    2147           0 :                         goto cleanup;
    2148           0 :                 if (!(t = find_table_or_view_on_scope(m, NULL, schema, table, "CATALOG", false))) {
    2149           0 :                         set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "%s", m->errstr + 6)); /* Skip error code */
    2150           0 :                         goto cleanup;
    2151             :                 }
    2152           0 :                 if (!insert_allowed(m, t, t->base.name, "APPEND", "append")) {
    2153           0 :                         set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "%s", m->errstr + 6)); /* Skip error code */
    2154           0 :                         goto cleanup;
    2155             :                 }
    2156           0 :                 if ((t->s && t->s->parts && (pt = partition_find_part(m->session->tr, t, NULL))) || isRangePartitionTable(t) || isListPartitionTable(t)) {
    2157           0 :                         set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Appending to a table from a merge table hierarchy via 'monetdbe_append' is not possible at the moment"));
    2158           0 :                         goto cleanup;
    2159             :                 }
    2160           0 :                 if (t->idxs) {
    2161           0 :                         for (node *n = ol_first_node(t->idxs); n; n = n->next) {
    2162           0 :                                 sql_idx *i = n->data;
    2163             : 
    2164           0 :                                 if (i->key) {
    2165           0 :                                         set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append",
    2166             :                                                                 "Appending to a table with key constraints via 'monetdbe_append' is not possible at the moment"));
    2167           0 :                                         goto cleanup;
    2168           0 :                                 } else if (hash_index(i->type) && list_length(i->columns) > 1) {
    2169           0 :                                         set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append",
    2170             :                                                                 "Appending to a table with hash indexes referring to more than one column via 'monetdbe_append' is not possible at the moment"));
    2171           0 :                                         goto cleanup;
    2172           0 :                                 } else if (i->type == join_idx) {
    2173           0 :                                         set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append",
    2174             :                                                                 "Appending to a table with join indexes via 'monetdbe_append' is not possible at the moment"));
    2175           0 :                                         goto cleanup;
    2176             :                                 }
    2177             :                         }
    2178             :                 }
    2179           0 :                 if (t->triggers) {
    2180           0 :                         for (n = ol_first_node(t->triggers); n; n = n->next) {
    2181           0 :                                 sql_trigger *trigger = n->data;
    2182             : 
    2183           0 :                                 if (trigger->event == 0) { /* insert event */
    2184           0 :                                         set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append",
    2185             :                                                                 "Appending to a table with triggers at the insert event via 'monetdbe_append' is not possible at the moment"));
    2186           0 :                                         goto cleanup;
    2187             :                                 }
    2188             :                         }
    2189             :                 }
    2190             :         }
    2191             : 
    2192             :         /* for now no default values, ie user should supply all columns */
    2193           0 :         if (column_count != (size_t)ol_length(t->columns)) {
    2194           0 :                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Incorrect number of columns"));
    2195           0 :                 goto cleanup;
    2196             :         }
    2197             : 
    2198           0 :         cnt = input[0]->count;
    2199           0 :         if (store->storage_api.claim_tab(m->session->tr, t, cnt, &offset, &pos) != LOG_OK) {
    2200           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_append", "Claim failed"));
    2201           0 :                 goto cleanup;
    2202             :         }
    2203             :         /* signal an insert was made on the table */
    2204           0 :         if (!isNew(t) && isGlobal(t) && !isGlobalTemp(t) && sql_trans_add_dependency_change(m->session->tr, t->base.id, dml) != LOG_OK) {
    2205           0 :                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", MAL_MALLOC_FAIL));
    2206           0 :                 goto cleanup;
    2207             :         }
    2208             : 
    2209           0 :         for (i = 0, n = ol_first_node(t->columns); i < column_count && n; i++, n = n->next) {
    2210           0 :                 sql_column *c = n->data;
    2211           0 :                 int mtype = monetdbe_type(input[i]->type);
    2212           0 :                 const void* nil = (mtype>=0)?ATOMnilptr(mtype):NULL;
    2213           0 :                 char *v = input[i]->data;
    2214             : 
    2215           0 :                 if (mtype < 0) {
    2216           0 :                         set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Cannot find type for column %zu", i));
    2217           0 :                         goto cleanup;
    2218           0 :                 } else if (input[i]->count != cnt) {
    2219           0 :                         set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Number of values don't match between columns"));
    2220           0 :                         goto cleanup;
    2221             :                 }
    2222           0 :                 if (mtype >= TYPE_bit && mtype <=
    2223             : #ifdef HAVE_HGE
    2224             :         TYPE_hge
    2225             : #else
    2226             :         TYPE_lng
    2227             : #endif
    2228             :                 ) {
    2229             :                         //-------------------------------------
    2230             :                         BAT *bn = NULL;
    2231             : 
    2232           0 :                         if (mtype != c->type.type->localtype) {
    2233           0 :                                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Cannot append %d into column '%s'", input[i]->type, c->base.name));
    2234           0 :                                 goto cleanup;
    2235             :                         }
    2236             : 
    2237           0 :                         if ((bn = COLnew(0, mtype, 0, TRANSIENT)) == NULL) {
    2238           0 :                                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Cannot create append column"));
    2239           0 :                                 goto cleanup;
    2240             :                         }
    2241             : 
    2242             :                         //save prev heap pointer
    2243             :                         char *prev_base;
    2244             :                         size_t prev_size;
    2245           0 :                         prev_base = bn->theap->base;
    2246           0 :                         prev_size = bn->theap->size;
    2247             : 
    2248             :                         //BAT heap base to input[i]->data
    2249           0 :                         bn->theap->base = input[i]->data;
    2250           0 :                         bn->theap->size = tailsize(bn, cnt);
    2251             : 
    2252             :                         //BATsetdims(bn); called in COLnew
    2253           0 :                         BATsetcapacity(bn, cnt);
    2254           0 :                         BATsetcount(bn, cnt);
    2255             : 
    2256             :                         //set default flags
    2257           0 :                         BATsettrivprop(bn);
    2258             : 
    2259           0 :                         if (cnt > 1) {
    2260           0 :                                 bn->tsorted = bn->trevsorted = false;
    2261           0 :                                 bn->tnosorted = bn->tnorevsorted = 0;
    2262           0 :                                 bn->tkey = false;
    2263           0 :                                 bn->tnonil = false;
    2264           0 :                                 bn->tnil = false;
    2265             :                         }
    2266             : 
    2267           0 :                         if (store->storage_api.append_col(m->session->tr, c, offset, pos, bn, cnt, TYPE_bat) != 0) {
    2268           0 :                                 bn->theap->base = prev_base;
    2269           0 :                                 bn->theap->size = prev_size;
    2270           0 :                                 BBPreclaim(bn);
    2271           0 :                                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Cannot append BAT"));
    2272           0 :                                 goto cleanup;
    2273             :                         }
    2274             : 
    2275           0 :                         bn->theap->base = prev_base;
    2276           0 :                         bn->theap->size = prev_size;
    2277           0 :                         BBPreclaim(bn);
    2278           0 :                 } else if (mtype == TYPE_str) {
    2279             :                         char **d = (char**)v;
    2280             : 
    2281           0 :                         for (size_t j=0; j<cnt; j++) {
    2282           0 :                                 if (!d[j]) {
    2283           0 :                                         d[j] = (char*) nil;
    2284           0 :                                 } else if (!checkUTF8(d[j])) {
    2285           0 :                                         set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Incorrectly encoded UTF-8"));
    2286           0 :                                         goto cleanup;
    2287             :                                 }
    2288             :                         }
    2289           0 :                         if (store->storage_api.append_col(m->session->tr, c, offset, pos, d, cnt, mtype) != 0) {
    2290           0 :                                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Cannot append values"));
    2291           0 :                                 goto cleanup;
    2292             :                         }
    2293           0 :                 } else if (mtype == TYPE_timestamp) {
    2294             :                         int err = 0;
    2295           0 :                         timestamp *d = GDKmalloc(sizeof(timestamp)*cnt);
    2296           0 :                         if (!d) {
    2297           0 :                                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Cannot append values"));
    2298           0 :                                 goto cleanup;
    2299             :                         }
    2300             :                         monetdbe_data_timestamp* ts = (monetdbe_data_timestamp*)v;
    2301             : 
    2302           0 :                         for (size_t j=0; j<cnt; j++){
    2303           0 :                                 monetdbe_data_timestamp mdt = ts[j];
    2304             : 
    2305           0 :                                 if (timestamp_is_null(&mdt)) {
    2306           0 :                                         d[j] = *(timestamp*) nil;
    2307             :                                 } else {
    2308           0 :                                         d[j] = timestamp_from_data(&mdt);
    2309             :                                 }
    2310             :                         }
    2311           0 :                         if (store->storage_api.append_col(m->session->tr, c, offset, pos, d, cnt, mtype) != 0) {
    2312           0 :                                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Cannot append values"));
    2313             :                                 err = 1;
    2314             :                         }
    2315           0 :                         GDKfree(d);
    2316           0 :                         if (err)
    2317           0 :                                 goto cleanup;
    2318           0 :                 } else if (mtype == TYPE_date) {
    2319             :                         int err = 0;
    2320           0 :                         date *d = GDKmalloc(sizeof(date)*cnt);
    2321           0 :                         if (!d) {
    2322           0 :                                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Cannot append values"));
    2323           0 :                                 goto cleanup;
    2324             :                         }
    2325             :                         monetdbe_data_date* de = (monetdbe_data_date*)v;
    2326             : 
    2327           0 :                         for (size_t j=0; j<cnt; j++){
    2328           0 :                                 monetdbe_data_date mdt = de[j];
    2329             : 
    2330           0 :                                 if (date_is_null(&mdt)) {
    2331           0 :                                         d[j] = *(date*) nil;
    2332             :                                 } else {
    2333           0 :                                         d[j] = date_from_data(&mdt);
    2334             :                                 }
    2335             :                         }
    2336           0 :                         if (store->storage_api.append_col(m->session->tr, c, offset, pos, d, cnt, mtype) != 0) {
    2337           0 :                                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Cannot append values"));
    2338             :                                 err = 1;
    2339             :                         }
    2340           0 :                         GDKfree(d);
    2341           0 :                         if (err)
    2342           0 :                                 goto cleanup;
    2343           0 :                 } else if (mtype == TYPE_daytime) {
    2344             :                         int err = 0;
    2345           0 :                         daytime *d = GDKmalloc(sizeof(daytime)*cnt);
    2346           0 :                         if (!d) {
    2347           0 :                                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Cannot append values"));
    2348           0 :                                 goto cleanup;
    2349             :                         }
    2350             :                         monetdbe_data_time* t = (monetdbe_data_time*)v;
    2351             : 
    2352           0 :                         for (size_t j=0; j<cnt; j++){
    2353           0 :                                 monetdbe_data_time mdt = t[j];
    2354             : 
    2355           0 :                                 if (time_is_null(&mdt)) {
    2356           0 :                                         d[j] = *(daytime*) nil;
    2357             :                                 } else {
    2358           0 :                                         d[j] = time_from_data(&mdt);
    2359             :                                 }
    2360             :                         }
    2361           0 :                         if (store->storage_api.append_col(m->session->tr, c, offset, pos, d, cnt, mtype) != 0) {
    2362           0 :                                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Cannot append values"));
    2363             :                                 err = 1;
    2364             :                         }
    2365           0 :                         GDKfree(d);
    2366           0 :                         if (err)
    2367           0 :                                 goto cleanup;
    2368           0 :                 } else if (mtype == TYPE_blob) {
    2369             :                         int err = 0;
    2370             :                         size_t j = 0;
    2371           0 :                         blob **d = GDKmalloc(sizeof(blob*)*cnt);
    2372           0 :                         if (!d) {
    2373           0 :                                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Cannot append values"));
    2374           0 :                                 goto cleanup;
    2375             :                         }
    2376             :                         monetdbe_data_blob* be = (monetdbe_data_blob*)v;
    2377             : 
    2378           0 :                         for (j=0; j<cnt; j++){
    2379           0 :                                 if (blob_is_null(be+j)) {
    2380           0 :                                         d[j] = (blob*)nil;
    2381             :                                 } else {
    2382           0 :                                         size_t len = be[j].size;
    2383           0 :                                         var_t nlen = blobsize(len);
    2384           0 :                                         blob *b = (blob*)GDKmalloc(nlen);
    2385           0 :                                         if (!b) {
    2386           0 :                                                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_append", MAL_MALLOC_FAIL));
    2387             :                                                 err = 1;
    2388             :                                                 break;
    2389             :                                         }
    2390           0 :                                         b->nitems = len;
    2391           0 :                                         memcpy(b->data, be[j].data, len);
    2392           0 :                                         d[j] = b;
    2393             :                                 }
    2394             :                         }
    2395           0 :                         if (!err && store->storage_api.append_col(m->session->tr, c, offset, pos, d, cnt, mtype) != 0) {
    2396           0 :                                 set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "Cannot append values"));
    2397             :                                 err = 1;
    2398             :                         }
    2399           0 :                         for (size_t k=0; k<j; k++){
    2400           0 :                                 if (d[k] != nil)
    2401           0 :                                         GDKfree(d[k]);
    2402             :                         }
    2403           0 :                         GDKfree(d);
    2404           0 :                         if (err)
    2405           0 :                                 goto cleanup;
    2406             :                 } else {
    2407           0 :                         set_error(mdbe, createException(SQL, "monetdbe.monetdbe_append", "The internal type '%s' is not supported on monetdbe append at the moment", ATOMname(mtype)));
    2408           0 :                         goto cleanup;
    2409             :                 }
    2410             :         }
    2411             : 
    2412           0 :         if (mdbe->mid) {
    2413             :                 char nme[16];
    2414           0 :                 const char *name        = number2name(nme, sizeof(nme), ++((backend*)  mdbe->c->sqlcontext)->remote);
    2415             :                 Symbol prg; // local program
    2416             : 
    2417           0 :                 if ( (prg = newFunction(userRef, putName(name), FUNCTIONsymbol)) == NULL ) {
    2418           0 :                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_append", MAL_MALLOC_FAIL));
    2419           0 :                         goto cleanup;
    2420             :                 }
    2421             : 
    2422           0 :                 MalBlkPtr mb = prg->def;
    2423           0 :                 InstrPtr f = getInstrPtr(mb, 0);
    2424           0 :                 f->retc = f->argc = 0;
    2425             : 
    2426           0 :                 InstrPtr r = newFcnCall(mb, remoteRef, registerRef);
    2427           0 :                 setArgType(mb, r, 0, TYPE_str);
    2428           0 :                 r = pushStr(mb, r, mdbe->mid);
    2429           0 :                 r = pushStr(mb, r, userRef);
    2430           0 :                 r = pushStr(mb, r, putName(remote_prg->name));
    2431             : 
    2432           0 :                 InstrPtr e = newInstructionArgs(mb, remoteRef, execRef, 4 + ol_length(t->columns));
    2433           0 :                 setDestVar(e, newTmpVariable(mb, TYPE_any));
    2434           0 :                 e = pushStr(mb, e, mdbe->mid);
    2435           0 :                 e = pushStr(mb, e, userRef);
    2436           0 :                 e = pushArgument(mb, e, getArg(r, 0));
    2437             : 
    2438           0 :                 for (i = 0, n = ol_first_node(t->columns); i < (unsigned) ol_length(t->columns); i++, n = n->next) {
    2439           0 :                         sql_column *c = n->data;
    2440           0 :                         BAT* b = store->storage_api.bind_col(m->session->tr, c, RDONLY);
    2441           0 :                         if (b == NULL) {
    2442           0 :                                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_append", MAL_MALLOC_FAIL));
    2443           0 :                                 freeSymbol(prg);
    2444           0 :                                 goto cleanup;
    2445             :                         }
    2446             : 
    2447           0 :                         int idx = newTmpVariable(mb, newBatType(c->type.type->localtype));
    2448           0 :                         ValRecord v = { .vtype = TYPE_bat, .len = ATOMlen(TYPE_bat, &b->batCacheid), .val.bval = b->batCacheid};
    2449           0 :                         getVarConstant(mb, idx) = v;
    2450           0 :                         setVarConstant(mb, idx);
    2451           0 :                         BBPunfix(b->batCacheid);
    2452             : 
    2453           0 :                         InstrPtr p = newFcnCall(mb, remoteRef, putRef);
    2454             :                         ;
    2455           0 :                         setArgType(mb, p, 0, TYPE_str);
    2456           0 :                         p = pushStr(mb, p, mdbe->mid);
    2457           0 :                         p = pushArgument(mb, p, idx);
    2458             : 
    2459           0 :                         e = pushArgument(mb, e, getArg(p, 0));
    2460             :                 }
    2461             : 
    2462           0 :                 pushInstruction(mb, e);
    2463             : 
    2464           0 :                 InstrPtr ri = newInstruction(mb, NULL, NULL);
    2465           0 :                 ri->barrier= RETURNsymbol;
    2466           0 :                 ri->retc = ri->argc = 0;
    2467           0 :                 pushInstruction(mb, ri);
    2468             : 
    2469           0 :                 if ( (mdbe->msg = chkProgram(mdbe->c->usermodule, mb)) != MAL_SUCCEED ) {
    2470           0 :                         freeSymbol(prg);
    2471           0 :                         goto cleanup;
    2472             :                 }
    2473             : 
    2474           0 :                 mdbe->msg = runMAL(mdbe->c, mb, 0, NULL);
    2475           0 :                 freeSymbol(prg);
    2476             :         }
    2477             : 
    2478           0 : cleanup:
    2479           0 :         if (pos)
    2480           0 :                 BBPreclaim(pos);
    2481           0 :         mdbe->msg = commit_action(m, mdbe, NULL, NULL);
    2482           0 :         return mdbe->msg;
    2483             : }
    2484             : 
    2485             : const void *
    2486           0 : monetdbe_null(monetdbe_database dbhdl, monetdbe_types t)
    2487             : {
    2488             :         monetdbe_database_internal *mdbe = (monetdbe_database_internal*)dbhdl;
    2489           0 :         int mtype = monetdbe_type(t);
    2490             : 
    2491           0 :         if (mtype < 0)
    2492             :                 return NULL;
    2493             : 
    2494           0 :         if ((mtype >= TYPE_bit && mtype <=
    2495             : #ifdef HAVE_HGE
    2496             :         TYPE_hge
    2497             : #else
    2498             :         TYPE_lng
    2499             : #endif
    2500             :                         ))
    2501           0 :                 return ATOMnilptr(mtype);
    2502           0 :         else if (mtype == TYPE_str)
    2503             :                 return NULL;
    2504           0 :         else if (mtype == TYPE_blob)
    2505           0 :                 return &mdbe->blob_null;
    2506           0 :         else if (mtype == TYPE_date)
    2507           0 :                 return &mdbe->date_null;
    2508           0 :         else if (mtype == TYPE_daytime)
    2509           0 :                 return &mdbe->time_null;
    2510           0 :         else if (mtype == TYPE_timestamp)
    2511           0 :                 return &mdbe->timestamp_null;
    2512             :         return NULL;
    2513             : }
    2514             : 
    2515             : char*
    2516           4 : monetdbe_result_fetch(monetdbe_result* mres, monetdbe_column** res, size_t column_index)
    2517             : {
    2518             :         BAT* b = NULL;
    2519             :         int bat_type;
    2520             :         mvc* m;
    2521             :         monetdbe_result_internal* result = (monetdbe_result_internal*) mres;
    2522             :         sql_subtype* sqltpe = NULL;
    2523             :         monetdbe_column* column_result = NULL;
    2524             :         size_t j = 0;
    2525           4 :         monetdbe_database_internal *mdbe = result->mdbe;
    2526           4 :         Client c = mdbe->c;
    2527             : 
    2528             : 
    2529           4 :         if ((mdbe->msg = validate_database_handle(mdbe, "monetdbe.monetdbe_result_fetch")) != MAL_SUCCEED) {
    2530             : 
    2531             :                 return mdbe->msg;
    2532             :         }
    2533             : 
    2534           4 :         if ((mdbe->msg = getSQLContext(c, NULL, &m, NULL)) != MAL_SUCCEED)
    2535           0 :                 goto cleanup;
    2536           4 :         if (!res) {
    2537           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_result_fetch", "Parameter res is NULL"));
    2538           0 :                 goto cleanup;
    2539             :         }
    2540           4 :         if (column_index >= mres->ncols) {
    2541           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_result_fetch", "Index out of range"));
    2542           0 :                 goto cleanup;
    2543             :         }
    2544             :         // check if we have the column converted already
    2545           4 :         if (result->converted_columns[column_index]) {
    2546           2 :                 *res = result->converted_columns[column_index];
    2547             : 
    2548           2 :                 return MAL_SUCCEED;
    2549             :         }
    2550             : 
    2551             :         // otherwise we have to convert the column
    2552           2 :         b = BATdescriptor(result->monetdbe_resultset->cols[column_index].b);
    2553           2 :         if (!b) {
    2554           0 :                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_result_fetch", RUNTIME_OBJECT_MISSING));
    2555           0 :                 goto cleanup;
    2556             :         }
    2557           2 :         bat_type = b->ttype;
    2558           2 :         sqltpe = &result->monetdbe_resultset->cols[column_index].type;
    2559             : 
    2560           2 :         if (bat_type == TYPE_bit) {
    2561           0 :                 GENERATE_BAT_INPUT(b, int8_t, bool, bit);
    2562           2 :         } else if (bat_type == TYPE_bte) {
    2563           0 :                 GENERATE_BAT_INPUT(b, int8_t, int8_t, bte);
    2564           2 :         } else if (bat_type == TYPE_sht) {
    2565           0 :                 GENERATE_BAT_INPUT(b, int16_t, int16_t, sht);
    2566           2 :         } else if (bat_type == TYPE_int) {
    2567           3 :                 GENERATE_BAT_INPUT(b, int32_t, int32_t, int);
    2568           1 :         } else if (bat_type == TYPE_oid) {
    2569           0 :                 GENERATE_BAT_INPUT(b, size_t, size_t, oid);
    2570           1 :         } else if (bat_type == TYPE_lng) {
    2571           0 :                 GENERATE_BAT_INPUT(b, int64_t, int64_t, lng);
    2572             : #ifdef HAVE_HGE
    2573           1 :         } else if (bat_type == TYPE_hge) {
    2574           0 :                 GENERATE_BAT_INPUT(b, __int128, int128_t, hge);
    2575             : #endif
    2576           1 :         } else if (bat_type == TYPE_flt) {
    2577           0 :                 GENERATE_BAT_INPUT(b, float, float, flt);
    2578           1 :         } else if (bat_type == TYPE_dbl) {
    2579           0 :                 GENERATE_BAT_INPUT(b, double, double, dbl);
    2580           1 :         } else if (bat_type == TYPE_str) {
    2581             :                 BATiter li;
    2582             :                 BUN p = 0, q = 0;
    2583           1 :                 GENERATE_BAT_INPUT_BASE(str);
    2584           1 :                 bat_data->count = (size_t) mres->nrows;
    2585           1 :                 if (bat_data->count) {
    2586           1 :                         bat_data->data = GDKzalloc(sizeof(char *) * bat_data->count);
    2587           1 :                         bat_data->null_value = NULL;
    2588           1 :                         if (!bat_data->data) {
    2589           0 :                                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_result_fetch", MAL_MALLOC_FAIL));
    2590           0 :                                 goto cleanup;
    2591             :                         }
    2592             :                 }
    2593             : 
    2594             :                 j = 0;
    2595           1 :                 li = bat_iterator(b);
    2596           3 :                 BATloop(b, p, q)
    2597             :                 {
    2598           2 :                         char *t = (char *)BUNtail(li, p);
    2599           2 :                         if (strcmp(t, str_nil) == 0) {
    2600           0 :                                 bat_data->data[j] = NULL;
    2601             :                         } else {
    2602           2 :                                 bat_data->data[j] = GDKstrdup(t);
    2603           2 :                                 if (!bat_data->data[j]) {
    2604           0 :                                         goto cleanup;
    2605             :                                 }
    2606             :                         }
    2607           2 :                         j++;
    2608             :                 }
    2609           1 :                 bat_iterator_end(&li);
    2610           0 :         } else if (bat_type == TYPE_date) {
    2611             :                 date *baseptr;
    2612           0 :                 GENERATE_BAT_INPUT_BASE(date);
    2613           0 :                 bat_data->count = (size_t) mres->nrows;
    2614           0 :                 if (bat_data->count) {
    2615           0 :                         bat_data->data = GDKmalloc(sizeof(bat_data->null_value) * bat_data->count);
    2616           0 :                         if (!bat_data->data) {
    2617           0 :                                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_result_fetch", MAL_MALLOC_FAIL));
    2618           0 :                                 goto cleanup;
    2619             :                         }
    2620             :                 }
    2621             : 
    2622           0 :                 baseptr = (date *)Tloc(b, 0);
    2623           0 :                 for (j = 0; j < bat_data->count; j++)
    2624           0 :                         data_from_date(baseptr[j], bat_data->data + j);
    2625           0 :                 memcpy(&bat_data->null_value, &mdbe->date_null, sizeof(monetdbe_data_date));
    2626           0 :         } else if (bat_type == TYPE_daytime) {
    2627             :                 daytime *baseptr;
    2628           0 :                 GENERATE_BAT_INPUT_BASE(time);
    2629           0 :                 bat_data->count = (size_t) mres->nrows;
    2630           0 :                 if (bat_data->count) {
    2631           0 :                         bat_data->data = GDKmalloc(sizeof(bat_data->null_value) * bat_data->count);
    2632           0 :                         if (!bat_data->data) {
    2633           0 :                                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_result_fetch", MAL_MALLOC_FAIL));
    2634           0 :                                 goto cleanup;
    2635             :                         }
    2636             :                 }
    2637             : 
    2638           0 :                 baseptr = (daytime *)Tloc(b, 0);
    2639           0 :                 for (j = 0; j < bat_data->count; j++)
    2640           0 :                         data_from_time(baseptr[j], bat_data->data + j);
    2641           0 :                 memcpy(&bat_data->null_value, &mdbe->time_null, sizeof(monetdbe_data_time));
    2642           0 :         } else if (bat_type == TYPE_timestamp) {
    2643             :                 timestamp *baseptr;
    2644           0 :                 GENERATE_BAT_INPUT_BASE(timestamp);
    2645           0 :                 bat_data->count = (size_t) mres->nrows;
    2646           0 :                 if (bat_data->count) {
    2647           0 :                         bat_data->data = GDKmalloc(sizeof(bat_data->null_value) * bat_data->count);
    2648           0 :                         if (!bat_data->data) {
    2649           0 :                                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_result_fetch", MAL_MALLOC_FAIL));
    2650           0 :                                 goto cleanup;
    2651             :                         }
    2652             :                 }
    2653             : 
    2654           0 :                 baseptr = (timestamp *)Tloc(b, 0);
    2655           0 :                 for (j = 0; j < bat_data->count; j++)
    2656           0 :                         data_from_timestamp(baseptr[j], bat_data->data + j);
    2657           0 :                 memcpy(&bat_data->null_value, &mdbe->timestamp_null, sizeof(monetdbe_data_timestamp));
    2658           0 :         } else if (bat_type == TYPE_blob) {
    2659             :                 BATiter li;
    2660             :                 BUN p = 0, q = 0;
    2661           0 :                 GENERATE_BAT_INPUT_BASE(blob);
    2662           0 :                 bat_data->count = (size_t) mres->nrows;
    2663           0 :                 if (bat_data->count) {
    2664           0 :                         bat_data->data = GDKmalloc(sizeof(monetdbe_data_blob) * bat_data->count);
    2665           0 :                         if (!bat_data->data) {
    2666           0 :                                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_result_fetch", MAL_MALLOC_FAIL));
    2667           0 :                                 goto cleanup;
    2668             :                         }
    2669             :                 }
    2670             :                 j = 0;
    2671             : 
    2672           0 :                 li = bat_iterator(b);
    2673           0 :                 BATloop(b, p, q)
    2674             :                 {
    2675           0 :                         blob *t = (blob *)BUNtail(li, p);
    2676           0 :                         if (t->nitems == ~(size_t)0) {
    2677           0 :                                 bat_data->data[j].size = 0;
    2678           0 :                                 bat_data->data[j].data = NULL;
    2679             :                         } else {
    2680           0 :                                 bat_data->data[j].size = t->nitems;
    2681           0 :                                 bat_data->data[j].data = GDKmalloc(t->nitems);
    2682           0 :                                 if (!bat_data->data[j].data) {
    2683           0 :                                         bat_iterator_end(&li);
    2684           0 :                                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_result_fetch", MAL_MALLOC_FAIL));
    2685           0 :                                         goto cleanup;
    2686             :                                 }
    2687           0 :                                 memcpy(bat_data->data[j].data, t->data, t->nitems);
    2688             :                         }
    2689           0 :                         j++;
    2690             :                 }
    2691           0 :                 bat_iterator_end(&li);
    2692           0 :                 bat_data->null_value.size = 0;
    2693           0 :                 bat_data->null_value.data = NULL;
    2694             :         } else {
    2695             :                 // unsupported type: convert to string
    2696             :                 BATiter li;
    2697             :                 BUN p = 0, q = 0;
    2698           0 :                 GENERATE_BAT_INPUT_BASE(str);
    2699           0 :                 bat_data->count = (size_t) mres->nrows;
    2700           0 :                 if (bat_data->count) {
    2701           0 :                         bat_data->null_value = NULL;
    2702           0 :                         bat_data->data = GDKzalloc(sizeof(char *) * bat_data->count);
    2703           0 :                         if (!bat_data->data) {
    2704           0 :                                 set_error(mdbe, createException(MAL, "monetdbe.monetdbe_result_fetch", MAL_MALLOC_FAIL));
    2705           0 :                                 goto cleanup;
    2706             :                         }
    2707             :                 }
    2708             :                 j = 0;
    2709             : 
    2710           0 :                 li = bat_iterator(b);
    2711           0 :                 BATloop(b, p, q)
    2712             :                 {
    2713           0 :                         void *t = BUNtail(li, p);
    2714           0 :                         if (BATatoms[bat_type].atomCmp(t, BATatoms[bat_type].atomNull) == 0) {
    2715           0 :                                 bat_data->data[j] = NULL;
    2716             :                         } else {
    2717           0 :                                 char *sresult = NULL;
    2718           0 :                                 size_t length = 0;
    2719           0 :                                 if (BATatoms[bat_type].atomToStr(&sresult, &length, t, true) == 0) {
    2720           0 :                                         bat_iterator_end(&li);
    2721           0 :                                         set_error(mdbe, createException(MAL, "monetdbe.monetdbe_result_fetch", "Failed to convert element to string"));
    2722           0 :                                         goto cleanup;
    2723             :                                 }
    2724           0 :                                 bat_data->data[j] = sresult;
    2725             :                         }
    2726           0 :                         j++;
    2727             :                 }
    2728           0 :                 bat_iterator_end(&li);
    2729             :         }
    2730           2 :         if (column_result)
    2731           2 :                 column_result->name = result->monetdbe_resultset->cols[column_index].name;
    2732           0 : cleanup:
    2733           2 :         if (b)
    2734           2 :                 BBPunfix(b->batCacheid);
    2735           2 :         if (mdbe->msg) {
    2736           0 :                 if (res)
    2737           0 :                         *res = NULL;
    2738           0 :                 monetdbe_destroy_column(column_result);
    2739           2 :         } else if (res) {
    2740           2 :                 result->converted_columns[column_index] = column_result;
    2741           2 :                 *res = result->converted_columns[column_index];
    2742             :         }
    2743           2 :         mdbe->msg = commit_action(m, mdbe, NULL, NULL);
    2744             : 
    2745           2 :         return mdbe->msg;
    2746             : }
    2747             : 
    2748             : static void
    2749           1 : data_from_date(date d, monetdbe_data_date *ptr)
    2750             : {
    2751           1 :         ptr->day = date_day(d);
    2752           1 :         ptr->month = date_month(d);
    2753           1 :         ptr->year = date_year(d);
    2754           1 : }
    2755             : 
    2756             : static date
    2757           0 : date_from_data(monetdbe_data_date *ptr)
    2758             : {
    2759           0 :         return date_create(ptr->year, ptr->month, ptr->day);
    2760             : }
    2761             : 
    2762             : static void
    2763           1 : data_from_time(daytime d, monetdbe_data_time *ptr)
    2764             : {
    2765           1 :         ptr->hours = daytime_hour(d);
    2766           1 :         ptr->minutes = daytime_min(d);
    2767           1 :         ptr->seconds = daytime_sec(d);
    2768           1 :         ptr->ms = daytime_usec(d) / 1000;
    2769           1 : }
    2770             : 
    2771             : static daytime
    2772           0 : time_from_data(monetdbe_data_time *ptr)
    2773             : {
    2774           0 :         return daytime_create(ptr->hours, ptr->minutes, ptr->seconds, ptr->ms * 1000);
    2775             : }
    2776             : 
    2777             : static void
    2778           1 : data_from_timestamp(timestamp d, monetdbe_data_timestamp *ptr)
    2779             : {
    2780           1 :         daytime tm = timestamp_daytime(d);
    2781           1 :         date dt = timestamp_date(d);
    2782             : 
    2783           1 :         ptr->date.day = date_day(dt);
    2784           1 :         ptr->date.month = date_month(dt);
    2785           1 :         ptr->date.year = date_year(dt);
    2786           1 :         ptr->time.hours = daytime_hour(tm);
    2787           1 :         ptr->time.minutes = daytime_min(tm);
    2788           1 :         ptr->time.seconds = daytime_sec(tm);
    2789           1 :         ptr->time.ms = daytime_usec(tm) / 1000;
    2790           1 : }
    2791             : 
    2792             : static timestamp
    2793           0 : timestamp_from_data(monetdbe_data_timestamp *ptr)
    2794             : {
    2795           0 :         return timestamp_create(
    2796           0 :                 date_create(ptr->date.year, ptr->date.month, ptr->date.day),
    2797           0 :                 daytime_create(ptr->time.hours, ptr->time.minutes, ptr->time.seconds, ptr->time.ms * 1000));
    2798             : }

Generated by: LCOV version 1.14