LCOV - code coverage report
Current view: top level - monetdb5/modules/mal - manifold.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 112 132 84.8 %
Date: 2021-10-13 02:24:04 Functions: 5 5 100.0 %

          Line data    Source code
       1             : /*
       2             :  * This Source Code Form is subject to the terms of the Mozilla Public
       3             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       4             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       5             :  *
       6             :  * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V.
       7             :  */
       8             : 
       9             : /*
      10             :  * M.L. Kersten
      11             :  */
      12             : #include "monetdb_config.h"
      13             : #include "manifold.h"
      14             : #include "mal_resolve.h"
      15             : #include "mal_builder.h"
      16             : 
      17             : /* The default iterator over known scalar commands.
      18             :  * It can be less efficient then the vector based implementations,
      19             :  * but saves quite some hacking in non-essential cases or
      20             :  * expensive user defined functions.
      21             :  *
      22             :  * To keep things simple and reasonably performant we limit the
      23             :  * implementation to those cases where a single BAT is returned.
      24             :  * Arguments may be of any type. The MAL signature should be a COMMAND.
      25             :  *
      26             :  * The functionality has been extended to also perform the manifold
      27             :  * over aligned BATs, provided the underlying scalar function carries
      28             :  * the 'manifold' property.
      29             :  */
      30             : 
      31             : typedef struct{
      32             :         BAT *b;
      33             :         void *first;
      34             :         void *last;
      35             :         int     size;
      36             :         int type;
      37             :         BUN cnt;
      38             :         BATiter bi;
      39             :         BUN  o;
      40             :         BUN  q;
      41             :         str *s;
      42             : } MULTIarg;
      43             : 
      44             : typedef struct{
      45             :         Client cntxt;
      46             :         MalBlkPtr mb;
      47             :         MalStkPtr stk;
      48             :         InstrPtr pci;
      49             :         int fvar,lvar;
      50             :         MULTIarg *args;
      51             : } MULTItask;
      52             : 
      53             : 
      54             : // Loop through the first BAT
      55             : // keep the last error message received
      56             : #define ManifoldLoop(Type, ...)                                                                                 \
      57             :         do {                                                                                                                            \
      58             :                 Type *v = (Type*) mut->args[0].first;                                                        \
      59             :                 for (;;) {                                                                                                              \
      60             :                         msg = (*mut->pci->fcn)(v, __VA_ARGS__);                                           \
      61             :                         if (msg) break;                                                                                         \
      62             :                         if (++oo == olimit)                                                                                     \
      63             :                                 break;                                                                                                  \
      64             :                         for( i = mut->fvar; i<= mut->lvar; i++) {                                      \
      65             :                                 if(ATOMstorage(mut->args[i].type) == TYPE_void ){            \
      66             :                                         args[i] = (void*)  &mut->args[i].o;                                      \
      67             :                                         mut->args[i].o++;                                                                    \
      68             :                                 } else if(mut->args[i].size == 0) {                                          \
      69             :                                         ;                                                                                                       \
      70             :                                 } else if(ATOMstorage(mut->args[i].type) < TYPE_str ) {   \
      71             :                                         args[i] += mut->args[i].size;                                                \
      72             :                                 } else if (ATOMvarsized(mut->args[i].type)) {                        \
      73             :                                         mut->args[i].o++;                                                                    \
      74             :                                         mut->args[i].s = (str *) BUNtail(mut->args[i].bi, mut->args[i].o); \
      75             :                                         args[i] = (void*)  &mut->args[i].s;                                      \
      76             :                                 } else {                                                                                                \
      77             :                                         mut->args[i].o++;                                                                    \
      78             :                                         mut->args[i].s = (str *) BUNtloc(mut->args[i].bi, mut->args[i].o); \
      79             :                                         args[i] = (void*)  &mut->args[i].s;                                      \
      80             :                                 }                                                                                                               \
      81             :                         }                                                                                                                       \
      82             :                         v++;                                                                                                            \
      83             :                 }                                                                                                                               \
      84             :         } while (0)
      85             : 
      86             : // The target BAT tail type determines the result variable
      87             : #ifdef HAVE_HGE
      88             : #define Manifoldbody_hge(...)                                   \
      89             :         case TYPE_hge: ManifoldLoop(hge,__VA_ARGS__); break
      90             : #else
      91             : #define Manifoldbody_hge(...)
      92             : #endif
      93             : #define Manifoldbody(...)                                                                                               \
      94             :         do {                                                                                                                            \
      95             :                 switch(ATOMstorage(mut->args[0].b->ttype)){                                               \
      96             :                 case TYPE_bte: ManifoldLoop(bte,__VA_ARGS__); break;                    \
      97             :                 case TYPE_sht: ManifoldLoop(sht,__VA_ARGS__); break;                    \
      98             :                 case TYPE_int: ManifoldLoop(int,__VA_ARGS__); break;                    \
      99             :                 case TYPE_lng: ManifoldLoop(lng,__VA_ARGS__); break;                    \
     100             :                 Manifoldbody_hge(__VA_ARGS__);                                                                  \
     101             :                 case TYPE_oid: ManifoldLoop(oid,__VA_ARGS__); break;                    \
     102             :                 case TYPE_flt: ManifoldLoop(flt,__VA_ARGS__); break;                    \
     103             :                 case TYPE_dbl: ManifoldLoop(dbl,__VA_ARGS__); break;                    \
     104             :                 case TYPE_uuid: ManifoldLoop(uuid,__VA_ARGS__); break;                  \
     105             :                 case TYPE_str:                                                                                                  \
     106             :                 default: {                                                                                                              \
     107             :                         for (;;) {                                                                                                      \
     108             :                                 msg = (*mut->pci->fcn)(&y, __VA_ARGS__);                              \
     109             :                                 if (msg)                                                                                                \
     110             :                                         break;                                                                                          \
     111             :                                 if (bunfastapp(mut->args[0].b, (void*) y) != GDK_SUCCEED) \
     112             :                                         goto bunins_failed;                                                                     \
     113             :                                 GDKfree(y); y = NULL;                                                                   \
     114             :                                 if (++oo == olimit)                                                                             \
     115             :                                         break;                                                                                          \
     116             :                                 for( i = mut->fvar; i<= mut->lvar; i++) {                              \
     117             :                                         if(ATOMstorage(mut->args[i].type) == TYPE_void ){    \
     118             :                                                 args[i] = (void*)  &mut->args[i].o;                              \
     119             :                                                 mut->args[i].o++;                                                            \
     120             :                                         } else if(mut->args[i].size == 0) {                                  \
     121             :                                                 ;                                                                                               \
     122             :                                         } else if (ATOMstorage(mut->args[i].type) < TYPE_str){ \
     123             :                                                 args[i] += mut->args[i].size;                                        \
     124             :                                         } else if(ATOMvarsized(mut->args[i].type)){                  \
     125             :                                                 mut->args[i].o++;                                                            \
     126             :                                                 mut->args[i].s = (str*) BUNtail(mut->args[i].bi, mut->args[i].o); \
     127             :                                                 args[i] =  (void*) & mut->args[i].s;                     \
     128             :                                         } else {                                                                                        \
     129             :                                                 mut->args[i].o++;                                                            \
     130             :                                                 mut->args[i].s = (str*) BUNtloc(mut->args[i].bi, mut->args[i].o); \
     131             :                                                 args[i] =  (void*) & mut->args[i].s;                     \
     132             :                                         }                                                                                                       \
     133             :                                 }                                                                                                               \
     134             :                         }                                                                                                                       \
     135             :                         break;                                                                                                          \
     136             :                 }                                                                                                                               \
     137             :                 }                                                                                                                               \
     138             :                 mut->args[0].b->theap->dirty = true;                                                   \
     139             :         } while (0)
     140             : 
     141             : // single argument is preparatory step for GDK_mapreduce
     142             : // Only the last error message is returned, the value of
     143             : // an erroneous call depends on the operator itself.
     144             : static str
     145         443 : MANIFOLDjob(MULTItask *mut)
     146             : {       int i;
     147             :         char **args;
     148         443 :         str y = NULL, msg= MAL_SUCCEED;
     149         443 :         oid oo = 0, olimit = mut->args[mut->fvar].cnt;
     150             : 
     151         443 :         if (olimit == 0)
     152             :                 return msg;                             /* nothing to do */
     153             : 
     154         415 :         args = (char**) GDKzalloc(sizeof(char*) * mut->pci->argc);
     155         415 :         if( args == NULL)
     156           0 :                 throw(MAL,"mal.manifold", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     157             : 
     158             :         // the mod.fcn arguments are ignored from the call
     159        1172 :         for( i = mut->pci->retc+2; i< mut->pci->argc; i++) {
     160         757 :                 if ( mut->args[i].b ){
     161         453 :                         if(ATOMstorage(mut->args[i].type) < TYPE_str){
     162          49 :                                 args[i] = (char*) mut->args[i].first;
     163         404 :                         } else if(ATOMvarsized(mut->args[i].type)){
     164         392 :                                 mut->args[i].s = (str*) BUNtail(mut->args[i].bi, mut->args[i].o);
     165         392 :                                 args[i] =  (void*) & mut->args[i].s;
     166             :                         } else {
     167          12 :                                 mut->args[i].s = (str*) BUNtloc(mut->args[i].bi, mut->args[i].o);
     168          12 :                                 args[i] =  (void*) & mut->args[i].s;
     169             :                         }
     170             :                 } else {
     171         304 :                         args[i] = (char *) getArgReference(mut->stk,mut->pci,i);
     172             :                 }
     173             :         }
     174             : 
     175             :         /* TRC_DEBUG(MAL_SERVER, "fvar %d lvar %d type %d\n", mut->fvar,mut->lvar, ATOMstorage(mut->args[mut->fvar].b->ttype));*/
     176             : 
     177             :         // use limited argument list expansion.
     178         415 :         switch(mut->pci->argc){
     179      220635 :         case 4: Manifoldbody(args[3]); break;
     180        1746 :         case 5: Manifoldbody(args[3],args[4]); break;
     181         180 :         case 6: Manifoldbody(args[3],args[4],args[5]); break;
     182         476 :         case 7: Manifoldbody(args[3],args[4],args[5],args[6]); break;
     183          13 :         case 8: Manifoldbody(args[3],args[4],args[5],args[6],args[7]); break;
     184           0 :         default:
     185           0 :                 msg= createException(MAL,"mal.manifold","manifold call limitation ");
     186             :         }
     187         415 :         if (ATOMextern(mut->args[0].type) && y)
     188           0 :                 GDKfree(y);
     189         415 : bunins_failed:
     190         415 :         GDKfree(args);
     191         415 :         return msg;
     192             : }
     193             : 
     194             : /* The manifold optimizer should check for the possibility
     195             :  * to use this implementation instead of the MAL loop.
     196             :  */
     197             : MALfcn
     198        3491 : MANIFOLDtypecheck(Client cntxt, MalBlkPtr mb, InstrPtr pci, int checkprops){
     199             :         int i, k, tpe= 0;
     200             :         InstrPtr q=0;
     201             :         MalBlkPtr nmb;
     202             :         MALfcn fcn;
     203             : 
     204        3491 :         if (pci->retc >1 || pci->argc > 8 || getModuleId(pci) == NULL) // limitation on MANIFOLDjob
     205             :                 return NULL;
     206             :         // We need a private MAL context to resolve the function call
     207        3453 :         nmb = newMalBlk(2 );
     208        3453 :         if( nmb == NULL)
     209             :                 return NULL;
     210             :         // the scalar function
     211        3453 :         q = newStmt(nmb,
     212        3453 :                 getVarConstant(mb,getArg(pci,pci->retc)).val.sval,
     213        3453 :                 getVarConstant(mb,getArg(pci,pci->retc+1)).val.sval);
     214             : 
     215             :         // Prepare the single result variable
     216        3453 :         tpe =getBatType(getArgType(mb,pci,0));
     217        3453 :         k= getArg(q,0);
     218        3453 :         setVarType(nmb,k,tpe);
     219        3453 :         if ( isVarFixed(nmb,k))
     220           0 :                 setVarFixed(nmb,k);
     221             : 
     222             :         // extract their scalar argument type
     223       10182 :         for ( i = pci->retc+2; i < pci->argc; i++){
     224        6729 :                 tpe = getBatType(getArgType(mb,pci,i));
     225        6729 :                 q= pushArgument(nmb,q, k= newTmpVariable(nmb, tpe));
     226        6729 :                 setVarFixed(nmb,k);
     227             :         }
     228             : 
     229             : /*
     230             :         TRC_DEBUG(MAL_SERVER, "Manifold operation\n");
     231             :         traceInstruction(MAL_SERVER, mb, 0, pci, LIST_MAL_ALL);
     232             :         traceInstruction(MAL_SERVER, nmb, 0, q, LIST_MAL_ALL);
     233             : */
     234             :         // Localize the underlying scalar operator
     235        3453 :         typeChecker(cntxt->usermodule, nmb, q, getPC(nmb, q), TRUE);
     236        3453 :         if (nmb->errors || q->fcn == NULL || q->token != CMDcall ||
     237         810 :                 (checkprops && q->blk && q->blk->unsafeProp) )
     238             :                 fcn = NULL;
     239             :         else {
     240             :                 fcn = q->fcn;
     241             :                 // retain the type detected
     242        2048 :                 if ( !isVarFixed(mb, getArg(pci,0)))
     243           0 :                         setVarType( mb, getArg(pci,0), newBatType(getArgType(nmb,q,0)) );
     244             :         }
     245             : 
     246             : /*
     247             :         TRC_DEBUG(MAL_SERVER, "Success? %s\n", (fcn == NULL? "no":"yes"));
     248             :         traceInstruction(MAL_SERVER, nmb, 0, q, LIST_MAL_ALL);
     249             : */
     250             : 
     251        3453 :         freeMalBlk(nmb);
     252        3453 :         return fcn;
     253             : }
     254             : 
     255             : /*
     256             :  * The manifold should support aligned BATs as well
     257             :  */
     258             : static str
     259         443 : MANIFOLDevaluate(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci){
     260             :         MULTItask mut;
     261             :         MULTIarg *mat;
     262             :         int i, tpe= 0;
     263             :         BUN cnt = 0;
     264         443 :         oid o = 0;
     265             :         str msg = MAL_SUCCEED;
     266             :         MALfcn fcn;
     267             : 
     268         443 :         fcn= MANIFOLDtypecheck(cntxt,mb,pci,0);
     269         443 :         if( fcn == NULL)
     270           0 :                 throw(MAL, "mal.manifold", "Illegal manifold function call");
     271             : 
     272         443 :         mat = (MULTIarg *) GDKzalloc(sizeof(MULTIarg) * pci->argc);
     273         443 :         if( mat == NULL)
     274           0 :                 throw(MAL, "mal.manifold", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     275             : 
     276             :         // mr-job structure preparation
     277         443 :         mut.fvar = mut.lvar = 0;
     278         443 :         mut.cntxt= cntxt;
     279         443 :         mut.mb= mb;
     280         443 :         mut.stk= stk;
     281         443 :         mut.args= mat;
     282         443 :         mut.pci = pci;
     283             : 
     284             :         // prepare iterators
     285        1246 :         for( i = pci->retc+2; i < pci->argc; i++){
     286         803 :                 if ( isaBatType(getArgType(mb,pci,i)) ){
     287         486 :                         mat[i].b = BATdescriptor( *getArgReference_bat(stk,pci,i));
     288         486 :                         if ( mat[i].b == NULL){
     289           0 :                                 msg = createException(MAL,"mal.manifold", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     290           0 :                                 goto wrapup;
     291             :                         }
     292         486 :                         mat[i].bi = bat_iterator(mat[i].b);
     293         486 :                         mat[i].type = tpe = getBatType(getArgType(mb,pci,i));
     294         486 :                         if (mut.fvar == 0){
     295         443 :                                 mut.fvar = i;
     296         443 :                                 cnt = BATcount(mat[i].b);
     297          43 :                         } else if (BATcount(mat[i].b)!= cnt){
     298           0 :                                 msg = createException(MAL,"mal.manifold","Columns must be of same length");
     299           0 :                                 goto wrapup;
     300             :                         }
     301         486 :                         mut.lvar = i;
     302         486 :                         mat[i].size = mat[i].bi.width;
     303         486 :                         mat[i].cnt = cnt;
     304         486 :                         if ( mat[i].b->ttype == TYPE_void){
     305           0 :                                 o = mat[i].b->tseqbase;
     306           0 :                                 mat[i].first = mat[i].last = (void*) &o;
     307             :                         } else {
     308         486 :                                 mat[i].first = (void*)  mat[i].bi.base;
     309         486 :                                 mat[i].last = (void *) ((char*) mat[i].bi.base + (BUNlast(mat[i].b) << mat[i].bi.shift));
     310             :                         }
     311         486 :                         mat[i].o = 0;
     312         486 :                         mat[i].q = BUNlast(mat[i].b);
     313             :                 } else {
     314         317 :                         mat[i].last = mat[i].first = (void *) getArgReference(stk,pci,i);
     315         317 :                         mat[i].type = getArgType(mb, pci, i);
     316             :                 }
     317             :         }
     318             : 
     319             :         // Then iterator over all BATs
     320         443 :         if( mut.fvar ==0){
     321           0 :                 msg= createException(MAL,"mal.manifold","At least one column required");
     322           0 :                 goto wrapup;
     323             :         }
     324             : 
     325             :         // prepare result variable
     326         443 :         mat[0].b =COLnew(mat[mut.fvar].b->hseqbase, getBatType(getArgType(mb,pci,0)), cnt, TRANSIENT);
     327         443 :         if ( mat[0].b == NULL){
     328           0 :                 msg= createException(MAL,"mal.manifold", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     329           0 :                 goto wrapup;
     330             :         }
     331         443 :         mat[0].b->tnonil=false;
     332         443 :         mat[0].b->tsorted=false;
     333         443 :         mat[0].b->trevsorted=false;
     334         443 :         mat[0].bi = (BATiter) {.b = NULL,};
     335         443 :         mat[0].first = (void *)  Tloc(mat[0].b, 0);
     336         443 :         mat[0].last = (void *)  Tloc(mat[0].b, BUNlast(mat[0].b));
     337             : 
     338         443 :         mut.pci = copyInstruction(pci);
     339         443 :         if ( mut.pci == NULL){
     340           0 :                 msg= createException(MAL,"mal.manifold", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     341           0 :                 goto wrapup;
     342             :         }
     343         443 :         mut.pci->fcn = fcn;
     344         443 :         msg = MANIFOLDjob(&mut);
     345         443 :         freeInstruction(mut.pci);
     346             : 
     347         443 : wrapup:
     348             :         // restore the argument types
     349        2132 :         for (i = pci->retc; i < pci->argc; i++){
     350        1689 :                 if ( mat[i].b) {
     351         486 :                         bat_iterator_end(&mat[i].bi);
     352         486 :                         BBPunfix(mat[i].b->batCacheid);
     353             :                 }
     354             :         }
     355         443 :         if (msg) {
     356           6 :                 BBPreclaim(mat[0].b);
     357             :         } else if (!msg) {
     358             :                 // consolidate the properties
     359         437 :                 if (ATOMstorage(mat[0].b->ttype) < TYPE_str)
     360         222 :                         BATsetcount(mat[0].b,cnt);
     361         437 :                 BATsettrivprop(mat[0].b);
     362         437 :                 BBPkeepref(*getArgReference_bat(stk,pci,0)=mat[0].b->batCacheid);
     363             :         }
     364         443 :         GDKfree(mat);
     365         443 :         return msg;
     366             : }
     367             : 
     368             : // The old code
     369             : static str
     370           1 : MANIFOLDremapMultiplex(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p)
     371             : {
     372             :     (void) mb;
     373             :     (void) cntxt;
     374           1 :     throw(MAL, "mal.multiplex", "Function '%s.%s' not defined",
     375           1 :                   *getArgReference_str(stk, p, p->retc),
     376           1 :                   *getArgReference_str(stk, p, p->retc + 1));
     377             : }
     378             : 
     379             : #include "mel.h"
     380             : mel_func manifold_init_funcs[] = {
     381             :  pattern("mal", "multiplex", MANIFOLDremapMultiplex, false, "", args(1,4, varargany("",0),arg("mod",str),arg("fcn",str),varargany("a",0))),
     382             :  pattern("batmal", "multiplex", MANIFOLDremapMultiplex, false, "", args(1,4, varargany("",0),arg("mod",str),arg("fcn",str),varargany("a",0))),
     383             :  pattern("mal", "manifold", MANIFOLDevaluate, false, "", args(1,4, batargany("",0),arg("mod",str),arg("fcn",str),varargany("a",0))),
     384             :  { .imp=NULL }
     385             : };
     386             : #include "mal_import.h"
     387             : #ifdef _MSC_VER
     388             : #undef read
     389             : #pragma section(".CRT$XCU",read)
     390             : #endif
     391         259 : LIB_STARTUP_FUNC(init_manifold_mal)
     392         259 : { mal_module("manifold", NULL, manifold_init_funcs); }

Generated by: LCOV version 1.14