LCOV - code coverage report
Current view: top level - sql/backends/monet5 - sql_optimizer.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 88 103 85.4 %
Date: 2021-10-13 02:24:04 Functions: 7 7 100.0 %

          Line data    Source code
       1             : /*
       2             :  * This Source Code Form is subject to the terms of the Mozilla Public
       3             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       4             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       5             :  *
       6             :  * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V.
       7             :  */
       8             : 
       9             : /*
      10             :  * N. Nes, M.L. Kersten
      11             :  * The queries are stored in the user cache after they have been
      12             :  * type checked and optimized.
      13             :  */
      14             : #include "monetdb_config.h"
      15             : #include "mal_builder.h"
      16             : #include "mal_debugger.h"
      17             : #include "mal_runtime.h"
      18             : #include "opt_prelude.h"
      19             : #include "sql_mvc.h"
      20             : #include "sql_optimizer.h"
      21             : #include "sql_scenario.h"
      22             : #include "sql_gencode.h"
      23             : #include "opt_pipes.h"
      24             : 
      25             : /* calculate the footprint for optimizer pipe line choices
      26             :  * and identify empty columns upfront for just in time optimizers.
      27             :  */
      28             : static lng
      29             : SQLgetColumnSize(sql_trans *tr, sql_column *c, int access)
      30             : {
      31     1004702 :         sqlstore *store = tr->store;
      32     1004702 :         return store->storage_api.count_col(tr, c, access);
      33             : }
      34             : 
      35             : static lng
      36             : SQLgetIdxSize(sql_trans *tr, sql_idx *i, int access)
      37             : {
      38        6679 :         sqlstore *store = tr->store;
      39        6679 :         return store->storage_api.count_idx(tr, i, access);
      40             : }
      41             : 
      42             : 
      43             : /*
      44             :  * The maximal space occupied by a query is calculated
      45             :  * under the assumption that the complete database should fit in memory.
      46             :  * The assumption is that the plan does not contain duplicate bind operations.
      47             :  * Calculation of the precise footprint is much more complex
      48             :  * and can not deal with intermediate structures, or fast
      49             :  * access using sorted probing.
      50             :  *
      51             :  * A run where we only take the size of a table only once,
      52             :  * caused major degration on SF100 Q3 with SSD(>6x)
      53             :  */
      54             : 
      55             : static lng
      56      356844 : SQLgetSpace(mvc *m, MalBlkPtr mb, int prepare)
      57             : {
      58      356844 :         sql_trans *tr = m->session->tr;
      59             :         lng size,space = 0, i;
      60             :         str lasttable = 0;
      61             : 
      62    12235720 :         for (i = 0; i < mb->stop; i++) {
      63    11878876 :                 InstrPtr p = mb->stmt[i];
      64             : 
      65             :                 /* now deal with the update binds, it is only necessary to identify that there are updats
      66             :                  * The actual size is not that important */
      67    11878876 :                 if (getModuleId(p) == sqlRef && getFunctionId(p) == bindRef  && p->retc <= 2){
      68     1004702 :                         char *sname = getVarConstant(mb, getArg(p, 1 + p->retc)).val.sval;
      69     1004702 :                         char *tname = getVarConstant(mb, getArg(p, 2 + p->retc)).val.sval;
      70     1004702 :                         char *cname = getVarConstant(mb, getArg(p, 3 + p->retc)).val.sval;
      71     1004702 :                         int access = getVarConstant(mb, getArg(p, 4 + p->retc)).val.ival;
      72     1004702 :                         sql_schema *s = mvc_bind_schema(m, sname);
      73             :                         sql_table *t = 0;
      74             :                         sql_column *c = 0;
      75             : 
      76     1004701 :                         if (!s)
      77           0 :                                 continue;
      78     1004701 :                         t = mvc_bind_table(m, s, tname);
      79     1004703 :                         if (!t || isDeclaredTable(t))
      80           0 :                                 continue;
      81     1004703 :                         c = mvc_bind_column(m, t, cname);
      82     1004702 :                         if (!c)
      83           0 :                                 continue;
      84             : 
      85             :                         /* we have to sum the cost of all three components of a BAT */
      86     1004702 :                         if (c && isTable(c->t) && (lasttable == 0 || strcmp(lasttable,tname)==0)) {
      87             :                                 size = SQLgetColumnSize(tr, c, access);
      88     1004702 :                                 space += size;  // accumulate once per table
      89             :                                 //lasttable = tname;     invalidate this attempt
      90     1004702 :                                 if( !prepare && size == 0  && ! t->system){
      91      198942 :                                         setFunctionId(p, emptybindRef);
      92             :                                 }
      93             :                         }
      94             :                 }
      95    11878876 :                 if (getModuleId(p) == sqlRef && (getFunctionId(p) == bindidxRef)) {
      96        6679 :                         char *sname = getVarConstant(mb, getArg(p, 1 + p->retc)).val.sval;
      97             :                         //char *tname = getVarConstant(mb, getArg(p, 2 + p->retc)).val.sval;
      98        6679 :                         char *idxname = getVarConstant(mb, getArg(p, 3 + p->retc)).val.sval;
      99        6679 :                         int access = getVarConstant(mb, getArg(p, 4 + p->retc)).val.ival;
     100        6679 :                         sql_schema *s = mvc_bind_schema(m, sname);
     101             : 
     102        6679 :                         if (getFunctionId(p) == bindidxRef) {
     103        6679 :                                 sql_idx *i = mvc_bind_idx(m, s, idxname);
     104             : 
     105        6679 :                                 if (i && isTable(i->t)) {
     106             :                                         size = SQLgetIdxSize(tr, i, access);
     107             : 
     108        6679 :                                         if( !prepare && size == 0 && ! i->t->system){
     109        3281 :                                                 setFunctionId(p, emptybindidxRef);
     110             :                                         }
     111             :                                 }
     112             :                         }
     113             :                 }
     114             :         }
     115      356844 :         return space;
     116             : }
     117             : 
     118             : /* gather the optimizer pipeline defined in the current session */
     119             : str
     120      475865 : getSQLoptimizer(mvc *m)
     121             : {
     122      475865 :         char *opt = get_string_global_var(m, "optimizer");
     123             :         char *pipe = "default_pipe";
     124             : 
     125      475865 :         if (opt)
     126             :                 pipe = opt;
     127      475865 :         return pipe;
     128             : }
     129             : 
     130             : static str
     131      356844 : addOptimizers(Client c, MalBlkPtr mb, char *pipe, int prepare)
     132             : {
     133             :         int i;
     134             :         InstrPtr q;
     135             :         backend *be;
     136             :         str msg= MAL_SUCCEED;
     137             : 
     138      356844 :         be = (backend *) c->sqlcontext;
     139      356844 :         assert(be && be->mvc);       /* SQL clients should always have their state set */
     140             : 
     141      356844 :         (void) SQLgetSpace(be->mvc, mb, prepare); // detect empty bats.
     142             :         /* The volcano optimizer seems relevant for traditional HDD settings.
     143             :          * It produced about 8 % improvement onf TPCH SF 100 on a 16G machine.
     144             :          * In a SSD setting it was counter productive, leading to worse parallel behavior.
     145             :          * The automatic switch to volcano is now disabled assuming more use of SSD.
     146             :          * The volcano optimizer pipeline can be used instead
     147             :         if(space && (pipe == NULL || strcmp(pipe,"default_pipe")== 0)){
     148             :                 if( space > (lng)(0.8 * MT_npages() * MT_pagesize())  && GDKnr_threads > 1){
     149             :                         pipe = "volcano_pipe";
     150             :                 }else
     151             :                         pipe = "default_pipe";
     152             :         } else
     153             :         */
     154      356844 :         pipe = pipe? pipe: "default_pipe";
     155      356844 :         msg = addOptimizerPipe(c, mb, pipe);
     156      356844 :         if (msg){
     157             :                 return msg;
     158             :         }
     159      356844 :         mb->keephistory |= be->mvc->emod & mod_debug;
     160      356844 :         if (be->no_mitosis) {
     161     3033806 :                 for (i = mb->stop - 1; i > 0; i--) {
     162     3033806 :                         q = getInstrPtr(mb, i);
     163     3033806 :                         if (q->token == ENDsymbol)
     164             :                                 break;
     165     2933376 :                         if (getFunctionId(q) == mitosisRef)
     166       95758 :                                 q->token = REMsymbol;        /* they are ignored */
     167             :                 }
     168             :         }
     169      356844 :         addtoMalBlkHistory(mb);
     170      356844 :         return msg;
     171             : }
     172             : 
     173             : /* Queries that should rely on the latest consolidated state
     174             :  * are not allowed to remove sql.binds operations.
     175             :  */
     176             : 
     177             : str
     178         715 : SQLoptimizeFunction(Client c, MalBlkPtr mb)
     179             : {
     180             :         str msg;
     181             :         str pipe;
     182         715 :         backend *be = (backend *) c->sqlcontext;
     183         715 :         assert(be && be->mvc);       /* SQL clients should always have their state set */
     184             : 
     185         715 :         pipe = getSQLoptimizer(be->mvc);
     186         715 :         msg = addOptimizers(c, mb, pipe, TRUE);
     187         715 :         if (msg)
     188             :                 return msg;
     189         715 :         mb->keephistory |= be->mvc->emod & mod_debug;
     190         715 :         msg = optimizeMALBlock(c, mb);
     191         715 :         mb->keephistory = FALSE;
     192         715 :         return msg;
     193             : }
     194             : 
     195             : str
     196      469298 : SQLoptimizeQuery(Client c, MalBlkPtr mb)
     197             : {
     198             :         backend *be;
     199             :         str msg = 0, pipe = 0;
     200             :         bool free_pipe = false;
     201      469298 :         InstrPtr p = mb->stmt[mb->stop -1];
     202             : 
     203      469298 :         if (p && mb->stop > 0 && getModuleId(p) == optimizerRef)
     204             :                 return MAL_SUCCEED; /* already optimized */
     205             : 
     206      356129 :         be = (backend *) c->sqlcontext;
     207      356129 :         assert(be && be->mvc);       /* SQL clients should always have their state set */
     208             : 
     209      356129 :         c->blkmode = 0;
     210      356129 :         msg = chkProgram(c->usermodule, mb);
     211             : 
     212             :         /*
     213             :          * An error in the compilation should be reported to the user.
     214             :          * And if the debugging option is set, the debugger is called
     215             :          * to allow inspection.
     216             :          */
     217      356129 :         if (msg != MAL_SUCCEED || mb->errors) {
     218           0 :                 if (c->listing)
     219           0 :                         printFunction(c->fdout, mb, 0, c->listing);
     220           0 :                 if (be->mvc->debug) {
     221           0 :                         str omsg = runMALDebugger(c, c->curprg->def);
     222           0 :                         if (omsg != MAL_SUCCEED)
     223           0 :                                 freeException(omsg); /* ignore error */
     224             :                 }
     225           0 :                 if (mb->errors && msg && msg != mb->errors) { /* if both set, throw mb->errors as the earliest one */
     226           0 :                         freeException(msg);
     227             :                         msg = MAL_SUCCEED;
     228             :                 }
     229           0 :                 str nmsg = createException(MAL, "optimizer.optimizeQuery", "%s", mb->errors ? mb->errors : msg);
     230           0 :                 freeException(msg);
     231           0 :                 return nmsg;
     232             :         }
     233             : 
     234      356129 :         pipe = getSQLoptimizer(be->mvc);
     235      356129 :         if( strcmp(pipe, "default_pipe") == 0 && strcmp(c->optimizer, "default_pipe") != 0) {
     236          15 :                 if (!(pipe = GDKstrdup(c->optimizer)))
     237           0 :                         throw(MAL, "sql.optimizeQuery", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     238             :                 free_pipe = true;
     239             :         }
     240             : 
     241      356129 :         msg = addOptimizers(c, mb, pipe, FALSE);
     242      356129 :         if (free_pipe)
     243          15 :                 GDKfree(pipe);
     244      356129 :         if (msg)
     245             :                 return msg;
     246      356129 :         mb->keephistory |= be->mvc->emod & mod_debug;
     247      356129 :         msg = optimizeMALBlock(c, mb);
     248      356129 :         return msg;
     249             : }
     250             : 
     251             : /* queries are added to the MAL catalog  under the client session namespace */
     252             : void
     253         836 : SQLaddQueryToCache(Client c)
     254             : {
     255         836 :         insertSymbol(c->usermodule, c->curprg);
     256         836 : }
     257             : 
     258             : void
     259           2 : SQLremoveQueryFromCache(Client c)
     260             : {
     261           2 :         deleteSymbol(c->usermodule, c->curprg);
     262           2 : }

Generated by: LCOV version 1.14