LCOV - code coverage report
Current view: top level - sql/server - rel_optimizer.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 4195 4902 85.6 %
Date: 2021-10-13 02:24:04 Functions: 185 190 97.4 %

          Line data    Source code
       1             : /*
       2             :  * This Source Code Form is subject to the terms of the Mozilla Public
       3             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       4             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       5             :  *
       6             :  * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V.
       7             :  */
       8             : 
       9             : #include "monetdb_config.h"
      10             : #include "rel_optimizer.h"
      11             : #include "rel_rel.h"
      12             : #include "rel_basetable.h"
      13             : #include "rel_exp.h"
      14             : #include "rel_prop.h"
      15             : #include "rel_dump.h"
      16             : #include "rel_select.h"
      17             : #include "rel_planner.h"
      18             : #include "rel_propagate.h"
      19             : #include "rel_rewriter.h"
      20             : #include "rel_remote.h"
      21             : #include "sql_mvc.h"
      22             : #include "sql_privileges.h"
      23             : #include "gdk_time.h"
      24             : 
      25             : typedef struct global_props {
      26             :         int cnt[ddl_maxops];
      27             :         uint8_t
      28             :                 has_mergetable:1,
      29             :                 needs_distinct:1;
      30             : } global_props;
      31             : 
      32             : static int
      33           0 : find_member_pos(list *l, sql_table *t)
      34             : {
      35             :         int i = 0;
      36           0 :         if (l) {
      37           0 :                 for (node *n = l->h; n ; n = n->next, i++) {
      38           0 :                         sql_part *pt = n->data;
      39           0 :                         if (pt->member == t->base.id)
      40           0 :                                 return i;
      41             :                 }
      42             :         }
      43             :         return -1;
      44             : }
      45             : 
      46             : /* The important task of the relational optimizer is to optimize the
      47             :    join order.
      48             : 
      49             :    The current implementation chooses the join order based on
      50             :    select counts, ie if one of the join sides has been reduced using
      51             :    a select this join is choosen over one without such selections.
      52             :  */
      53             : 
      54             : /* currently we only find simple column expressions */
      55             : sql_column *
      56     1202738 : name_find_column( sql_rel *rel, const char *rname, const char *name, int pnr, sql_rel **bt )
      57             : {
      58             :         sql_exp *alias = NULL;
      59             :         sql_column *c = NULL;
      60             : 
      61     2984269 :         switch (rel->op) {
      62      537467 :         case op_basetable: {
      63      537467 :                 sql_table *t = rel->l;
      64             : 
      65      537467 :                 if (rel->exps) {
      66             :                         sql_exp *e;
      67             : 
      68      537467 :                         if (rname)
      69      536675 :                                 e = exps_bind_column2(rel->exps, rname, name, NULL);
      70             :                         else
      71         792 :                                 e = exps_bind_column(rel->exps, name, NULL, NULL, 0);
      72      537467 :                         if (!e || e->type != e_column)
      73             :                                 return NULL;
      74      402593 :                         if (e->l)
      75             :                                 rname = e->l;
      76      402593 :                         name = e->r;
      77             :                 }
      78      402593 :                 if (rname && strcmp(t->base.name, rname) != 0)
      79             :                         return NULL;
      80      402593 :                 sql_table *mt = rel_base_get_mergetable(rel);
      81      402593 :                 if (ol_length(t->columns)) {
      82     1386936 :                         for (node *cn = ol_first_node(t->columns); cn; cn = cn->next) {
      83     1384709 :                                 sql_column *c = cn->data;
      84     1384709 :                                 if (strcmp(c->base.name, name) == 0) {
      85      400366 :                                         *bt = rel;
      86      400366 :                                         if (pnr < 0 || (mt &&
      87           0 :                                                 find_member_pos(mt->members, c->t) == pnr))
      88      400366 :                                                 return c;
      89             :                                 }
      90             :                         }
      91             :                 }
      92        2227 :                 if (name[0] == '%' && ol_length(t->idxs)) {
      93       17954 :                         for (node *cn = ol_first_node(t->idxs); cn; cn = cn->next) {
      94       17856 :                                 sql_idx *i = cn->data;
      95       17856 :                                 if (strcmp(i->base.name, name+1 /* skip % */) == 0) {
      96        2129 :                                         *bt = rel;
      97        2129 :                                         if (pnr < 0 || (mt &&
      98           0 :                                                 find_member_pos(mt->members, i->t) == pnr)) {
      99        2129 :                                                 sql_kc *c = i->columns->h->data;
     100        2129 :                                                 return c->c;
     101             :                                         }
     102             :                                 }
     103             :                         }
     104             :                 }
     105             :                 break;
     106             :         }
     107             :         case op_table:
     108             :                 /* table func */
     109             :                 return NULL;
     110           0 :         case op_ddl:
     111           0 :                 if (is_updateble(rel))
     112           0 :                         return name_find_column( rel->l, rname, name, pnr, bt);
     113             :                 return NULL;
     114      682005 :         case op_join:
     115             :         case op_left:
     116             :         case op_right:
     117             :         case op_full:
     118             :                 /* first right (possible subquery) */
     119      682005 :                 c = name_find_column( rel->r, rname, name, pnr, bt);
     120             :                 /* fall through */
     121             :         case op_semi:
     122             :         case op_anti:
     123      682005 :                 if (!c)
     124      510522 :                         c = name_find_column( rel->l, rname, name, pnr, bt);
     125             :                 return c;
     126      195786 :         case op_select:
     127             :         case op_topn:
     128             :         case op_sample:
     129      195786 :                 return name_find_column( rel->l, rname, name, pnr, bt);
     130       51136 :         case op_union:
     131             :         case op_inter:
     132             :         case op_except:
     133             : 
     134       51136 :                 if (pnr >= 0 || pnr == -2) {
     135             :                         /* first right (possible subquery) */
     136       51136 :                         c = name_find_column( rel->r, rname, name, pnr, bt);
     137       51136 :                         if (!c)
     138       42276 :                                 c = name_find_column( rel->l, rname, name, pnr, bt);
     139        8860 :                         return c;
     140             :                 }
     141             :                 return NULL;
     142             : 
     143     1512028 :         case op_project:
     144             :         case op_groupby:
     145     1512028 :                 if (!rel->exps)
     146             :                         break;
     147     1512028 :                 if (rname)
     148     1486142 :                         alias = exps_bind_column2(rel->exps, rname, name, NULL);
     149             :                 else
     150       25886 :                         alias = exps_bind_column(rel->exps, name, NULL, NULL, 1);
     151     1512028 :                 if (is_groupby(rel->op) && alias && alias->type == e_column && rel->r) {
     152      392403 :                         if (alias->l)
     153      380670 :                                 alias = exps_bind_column2(rel->r, alias->l, alias->r, NULL);
     154             :                         else
     155       11733 :                                 alias = exps_bind_column(rel->r, alias->r, NULL, NULL, 1);
     156             :                 }
     157     1512028 :                 if (is_groupby(rel->op) && !alias && rel->l) {
     158             :                         /* Group by column not found as alias in projection
     159             :                          * list, fall back to check plain input columns */
     160             :                         return name_find_column( rel->l, rname, name, pnr, bt);
     161             :                 }
     162             :                 break;
     163             :         case op_insert:
     164             :         case op_update:
     165             :         case op_delete:
     166             :         case op_truncate:
     167             :         case op_merge:
     168             :                 break;
     169             :         }
     170     1499773 :         if (alias) { /* we found an expression with the correct name, but
     171             :                         we need sql_columns */
     172     1100840 :                 if (rel->l && alias->type == e_column) /* real alias */
     173     1020594 :                         return name_find_column(rel->l, alias->l, alias->r, pnr, bt);
     174             :         }
     175             :         return NULL;
     176             : }
     177             : 
     178             : static sql_column *
     179      305559 : exp_find_column( sql_rel *rel, sql_exp *exp, int pnr )
     180             : {
     181      305559 :         if (exp->type == e_column) {
     182      288117 :                 sql_rel *bt = NULL;
     183      288117 :                 return name_find_column(rel, exp->l, exp->r, pnr, &bt);
     184             :         }
     185             :         return NULL;
     186             : }
     187             : 
     188             : static sql_column *
     189      180689 : exp_find_column_( sql_rel *rel, sql_exp *exp, int pnr, sql_rel **bt )
     190             : {
     191      180689 :         if (exp->type == e_column)
     192      180624 :                 return name_find_column(rel, exp->l, exp->r, pnr, bt);
     193             :         return NULL;
     194             : }
     195             : 
     196             : /* find column for the select/join expression */
     197             : static sql_column *
     198        5596 : sjexp_col(sql_exp *e, sql_rel *r)
     199             : {
     200             :         sql_column *res = NULL;
     201             : 
     202        5596 :         if (e->type == e_cmp && !is_complex_exp(e->flag)) {
     203        5596 :                 res = exp_find_column(r, e->l, -2);
     204        5596 :                 if (!res)
     205         487 :                         res = exp_find_column(r, e->r, -2);
     206             :         }
     207        5596 :         return res;
     208             : }
     209             : 
     210             : static sql_exp *
     211       14414 : list_find_exp( list *exps, sql_exp *e)
     212             : {
     213             :         sql_exp *ne = NULL;
     214             : 
     215       14414 :         if (e->type != e_column)
     216             :                 return NULL;
     217       14410 :         if (( e->l && (ne=exps_bind_column2(exps, e->l, e->r, NULL)) != NULL) ||
     218       13551 :            ((!e->l && (ne=exps_bind_column(exps, e->r, NULL, NULL, 1)) != NULL)))
     219         993 :                 return ne;
     220             :         return NULL;
     221             : }
     222             : 
     223             : static int
     224       24955 : kc_column_cmp(sql_kc *kc, sql_column *c)
     225             : {
     226             :         /* return on equality */
     227       24955 :         return !(c == kc->c);
     228             : }
     229             : 
     230             : static void psm_exps_properties(mvc *sql, global_props *gp, list *exps);
     231             : static void rel_properties(mvc *sql, global_props *gp, sql_rel *rel);
     232             : 
     233             : static void
     234      448975 : psm_exp_properties(mvc *sql, global_props *gp, sql_exp *e)
     235             : {
     236             :         /* only functions need fix up */
     237      537459 :         switch(e->type) {
     238             :         case e_column:
     239             :                 break;
     240      246421 :         case e_atom:
     241      246421 :                 if (e->f)
     242        4756 :                         psm_exps_properties(sql, gp, e->f);
     243             :                 break;
     244       48214 :         case e_convert:
     245       48214 :                 psm_exp_properties(sql, gp, e->l);
     246       48214 :                 break;
     247      133294 :         case e_aggr:
     248             :         case e_func:
     249      133294 :                 psm_exps_properties(sql, gp, e->l);
     250      133294 :                 assert(!e->r);
     251             :                 break;
     252           0 :         case e_cmp:
     253           0 :                 if (e->flag == cmp_or || e->flag == cmp_filter) {
     254           0 :                         psm_exps_properties(sql, gp, e->l);
     255           0 :                         psm_exps_properties(sql, gp, e->r);
     256           0 :                 } else if (e->flag == cmp_in || e->flag == cmp_notin) {
     257           0 :                         psm_exp_properties(sql, gp, e->l);
     258           0 :                         psm_exps_properties(sql, gp, e->r);
     259             :                 } else {
     260           0 :                         psm_exp_properties(sql, gp, e->l);
     261           0 :                         psm_exp_properties(sql, gp, e->r);
     262           0 :                         if (e->f)
     263             :                                 psm_exp_properties(sql, gp, e->f);
     264             :                 }
     265             :                 break;
     266      108264 :         case e_psm:
     267      108264 :                 if (e->flag & PSM_SET || e->flag & PSM_RETURN || e->flag & PSM_EXCEPTION) {
     268       40270 :                         psm_exp_properties(sql, gp, e->l);
     269       67994 :                 } else if (e->flag & PSM_WHILE || e->flag & PSM_IF) {
     270       11842 :                         psm_exp_properties(sql, gp, e->l);
     271       11842 :                         psm_exps_properties(sql, gp, e->r);
     272       11842 :                         if (e->flag == PSM_IF && e->f)
     273        1298 :                                 psm_exps_properties(sql, gp, e->f);
     274       56152 :                 } else if (e->flag & PSM_REL && e->l) {
     275       46189 :                         rel_properties(sql, gp, e->l);
     276             :                 }
     277             :                 break;
     278             :         }
     279      448975 : }
     280             : 
     281             : static void
     282      191395 : psm_exps_properties(mvc *sql, global_props *gp, list *exps)
     283             : {
     284             :         node *n;
     285             : 
     286      191395 :         if (!exps)
     287             :                 return;
     288      625860 :         for (n = exps->h; n; n = n->next)
     289      437133 :                 psm_exp_properties(sql, gp, n->data);
     290             : }
     291             : 
     292             : static void
     293     2608802 : rel_properties(mvc *sql, global_props *gp, sql_rel *rel)
     294             : {
     295     7707796 :         if (!rel)
     296             :                 return;
     297             : 
     298     7707791 :         gp->cnt[(int)rel->op]++;
     299     7707791 :         switch (rel->op) {
     300     1962273 :         case op_basetable:
     301             :         case op_table: {
     302     1962273 :                 if (is_basetable(rel->op)) {
     303     1934406 :                         sql_table *t = (sql_table *) rel->l;
     304             :                         sql_part *pt;
     305             : 
     306             :                         /* If the plan has a merge table or a child of one, then rel_merge_table_rewrite has to run */
     307     1937161 :                         gp->has_mergetable |= (isMergeTable(t) || (t->s && t->s->parts && (pt = partition_find_part(sql->session->tr, t, NULL))));
     308             :                 }
     309     1962273 :                 if (rel->op == op_table && rel->l && rel->flag != TRIGGER_WRAPPER)
     310             :                         rel_properties(sql, gp, rel->l);
     311             :         } break;
     312     1735571 :         case op_join:
     313             :         case op_left:
     314             :         case op_right:
     315             :         case op_full:
     316             : 
     317             :         case op_semi:
     318             :         case op_anti:
     319             : 
     320             :         case op_union:
     321             :         case op_inter:
     322             :         case op_except:
     323             : 
     324             :         case op_insert:
     325             :         case op_update:
     326             :         case op_delete:
     327             :         case op_merge:
     328     1735571 :                 if (rel->l)
     329     1735571 :                         rel_properties(sql, gp, rel->l);
     330     1735571 :                 if (rel->r)
     331             :                         rel_properties(sql, gp, rel->r);
     332             :                 break;
     333     3378445 :         case op_project:
     334             :         case op_select:
     335             :         case op_groupby:
     336             :         case op_topn:
     337             :         case op_sample:
     338             :         case op_truncate:
     339     3378445 :                 gp->needs_distinct |= need_distinct(rel);
     340     3378445 :                 if (rel->l)
     341             :                         rel_properties(sql, gp, rel->l);
     342             :                 break;
     343      631502 :         case op_ddl:
     344      631502 :                 if (rel->flag == ddl_psm && rel->exps)
     345       40205 :                         psm_exps_properties(sql, gp, rel->exps);
     346      631502 :                 if (rel->flag == ddl_output || rel->flag == ddl_create_seq || rel->flag == ddl_alter_seq || rel->flag == ddl_alter_table || rel->flag == ddl_create_table || rel->flag == ddl_create_view) {
     347       59086 :                         if (rel->l)
     348             :                                 rel_properties(sql, gp, rel->l);
     349      572416 :                 } else if (rel->flag == ddl_list || rel->flag == ddl_exception) {
     350        2028 :                         if (rel->l)
     351        2008 :                                 rel_properties(sql, gp, rel->l);
     352        2028 :                         if (rel->r)
     353             :                                 rel_properties(sql, gp, rel->r);
     354             :                 }
     355             :                 break;
     356             :         }
     357     2608802 : }
     358             : 
     359             : static sql_rel * rel_join_order(visitor *v, sql_rel *rel) ;
     360             : 
     361             : static void
     362      553895 : get_relations(visitor *v, sql_rel *rel, list *rels)
     363             : {
     364      767109 :         if (!rel_is_ref(rel) && rel->op == op_join && rel->exps == NULL) {
     365      213214 :                 sql_rel *l = rel->l;
     366      213214 :                 sql_rel *r = rel->r;
     367             : 
     368      213214 :                 get_relations(v, l, rels);
     369      213214 :                 get_relations(v, r, rels);
     370      213214 :                 rel->l = NULL;
     371      213214 :                 rel->r = NULL;
     372      213214 :                 rel_destroy(rel);
     373             :         } else {
     374      340681 :                 rel = rel_join_order(v, rel);
     375      340681 :                 append(rels, rel);
     376             :         }
     377      553895 : }
     378             : 
     379             : static void
     380      160492 : get_inner_relations(mvc *sql, sql_rel *rel, list *rels)
     381             : {
     382      267408 :         if (!rel_is_ref(rel) && is_join(rel->op)) {
     383      106916 :                 sql_rel *l = rel->l;
     384      106916 :                 sql_rel *r = rel->r;
     385             : 
     386      106916 :                 get_inner_relations(sql, l, rels);
     387             :                 get_inner_relations(sql, r, rels);
     388             :         } else {
     389      160492 :                 append(rels, rel);
     390             :         }
     391      160492 : }
     392             : 
     393             : static int
     394     3536597 : exp_count(int *cnt, sql_exp *e)
     395             : {
     396             :         int flag;
     397     3536597 :         if (!e)
     398             :                 return 0;
     399     3536597 :         if (find_prop(e->p, PROP_JOINIDX))
     400        7557 :                 *cnt += 100;
     401     3536597 :         if (find_prop(e->p, PROP_HASHCOL))
     402      568990 :                 *cnt += 100;
     403     3536597 :         if (find_prop(e->p, PROP_HASHIDX))
     404           0 :                 *cnt += 100;
     405     3536597 :         switch(e->type) {
     406     1258287 :         case e_cmp:
     407     1258287 :                 if (!is_complex_exp(e->flag)) {
     408     1135325 :                         exp_count(cnt, e->l);
     409     1135325 :                         exp_count(cnt, e->r);
     410     1135325 :                         if (e->f)
     411        7644 :                                 exp_count(cnt, e->f);
     412             :                 }
     413     1258287 :                 flag = e->flag;
     414     1258287 :                 switch (flag) {
     415     1080674 :                 case cmp_equal:
     416     1080674 :                         *cnt += 90;
     417     1080674 :                         return 90;
     418       20630 :                 case cmp_notequal:
     419       20630 :                         *cnt += 7;
     420       20630 :                         return 7;
     421       29018 :                 case cmp_gt:
     422             :                 case cmp_gte:
     423             :                 case cmp_lt:
     424             :                 case cmp_lte:
     425       29018 :                         *cnt += 6;
     426       29018 :                         if (e->l) {
     427             :                                 sql_exp *l = e->l;
     428       29018 :                                 sql_subtype *t = exp_subtype(l);
     429       29018 :                                 if (EC_TEMP(t->type->eclass)) /* give preference to temporal ranges */
     430         194 :                                         *cnt += 90;
     431             :                         }
     432       29018 :                         if (e->f){ /* range */
     433        7644 :                                 *cnt += 6;
     434        7644 :                                 return 12;
     435             :                         }
     436             :                         return 6;
     437         328 :                 case cmp_filter:
     438         328 :                         if (exps_card(e->r) > CARD_AGGR) {
     439             :                                 /* filters for joins are special */
     440          51 :                                 *cnt += 1000;
     441          51 :                                 return 1000;
     442             :                         }
     443         277 :                         *cnt += 2;
     444         277 :                         return 2;
     445      111972 :                 case cmp_in:
     446             :                 case cmp_notin: {
     447      111972 :                         list *l = e->r;
     448      111972 :                         int c = 9 - 10*list_length(l);
     449      111972 :                         *cnt += c;
     450      111972 :                         return c;
     451             :                 }
     452       10662 :                 case cmp_or: /* prefer or over functions */
     453       10662 :                         *cnt += 3;
     454       10662 :                         return 3;
     455             :                 case mark_in:
     456             :                 case mark_notin:
     457             :                         *cnt += 0;
     458             :                         return 0;
     459             :                 default:
     460             :                         return 0;
     461             :                 }
     462     1913047 :         case e_column:
     463     1913047 :                 *cnt += 20;
     464     1913047 :                 return 20;
     465      250864 :         case e_atom:
     466      250864 :                 *cnt += 10;
     467      250864 :                 return 10;
     468       13137 :         case e_func:
     469             :                 /* functions are more expensive, depending on the number of columns involved. */
     470       13137 :                 if (e->card == CARD_ATOM)
     471             :                         return 0;
     472       10890 :                 *cnt -= 5*list_length(e->l);
     473       10890 :                 return 5*list_length(e->l);
     474      101262 :         case e_convert:
     475             :                 /* functions are more expensive, depending on the number of columns involved. */
     476      101262 :                 if (e->card == CARD_ATOM)
     477             :                         return 0;
     478             :                 /* fall through */
     479             :         default:
     480       99372 :                 *cnt -= 5;
     481       99372 :                 return -5;
     482             :         }
     483             : }
     484             : 
     485             : static int
     486             : exp_keyvalue(sql_exp *e)
     487             : {
     488     1129769 :         int cnt = 0;
     489      254908 :         exp_count(&cnt, e);
     490     1129769 :         return cnt;
     491             : }
     492             : 
     493             : static sql_exp *
     494      543612 : joinexp_col(sql_exp *e, sql_rel *r)
     495             : {
     496      543612 :         if (e->type == e_cmp) {
     497      543612 :                 if (rel_has_exp(r, e->l) >= 0)
     498      275652 :                         return e->l;
     499      267960 :                 return e->r;
     500             :         }
     501           0 :         assert(0);
     502             :         return NULL;
     503             : }
     504             : 
     505             : static sql_column *
     506      406866 : table_colexp(sql_exp *e, sql_rel *r)
     507             : {
     508      406866 :         sql_table *t = r->l;
     509             : 
     510      406866 :         if (e->type == e_column) {
     511      381679 :                 const char *name = exp_name(e);
     512             :                 node *cn;
     513             : 
     514      381679 :                 if (r->exps) { /* use alias */
     515      831558 :                         for (cn = r->exps->h; cn; cn = cn->next) {
     516      831170 :                                 sql_exp *ce = cn->data;
     517      831170 :                                 if (strcmp(exp_name(ce), name) == 0) {
     518      381291 :                                         name = ce->r;
     519      381291 :                                         break;
     520             :                                 }
     521             :                         }
     522             :                 }
     523      989509 :                 for (cn = ol_first_node(t->columns); cn; cn = cn->next) {
     524      989121 :                         sql_column *c = cn->data;
     525      989121 :                         if (strcmp(c->base.name, name) == 0)
     526      381291 :                                 return c;
     527             :                 }
     528             :         }
     529             :         return NULL;
     530             : }
     531             : 
     532             : int
     533     1540370 : exp_joins_rels(sql_exp *e, list *rels)
     534             : {
     535             :         sql_rel *l = NULL, *r = NULL;
     536             : 
     537     1540370 :         assert (e->type == e_cmp);
     538             : 
     539     1540370 :         if (e->flag == cmp_or) {
     540             :                 l = NULL;
     541     1540370 :         } else if (e->flag == cmp_filter) {
     542           2 :                 list *ll = e->l;
     543           2 :                 list *lr = e->r;
     544             : 
     545           2 :                 l = find_rel(rels, ll->h->data);
     546           2 :                 r = find_rel(rels, lr->h->data);
     547     1540368 :         } else if (e->flag == cmp_in || e->flag == cmp_notin) {
     548           0 :                 list *lr = e->r;
     549             : 
     550           0 :                 l = find_rel(rels, e->l);
     551           0 :                 if (lr && lr->h)
     552           0 :                         r = find_rel(rels, lr->h->data);
     553             :         } else {
     554     1540368 :                 l = find_rel(rels, e->l);
     555     1540368 :                 r = find_rel(rels, e->r);
     556             :         }
     557             : 
     558     1540370 :         if (l && r)
     559      281270 :                 return 0;
     560             :         return -1;
     561             : }
     562             : 
     563             : static list *
     564      271988 : matching_joins(sql_allocator *sa, list *rels, list *exps, sql_exp *je)
     565             : {
     566             :         sql_rel *l, *r;
     567             : 
     568      271988 :         assert (je->type == e_cmp);
     569             : 
     570      271988 :         l = find_rel(rels, je->l);
     571      271988 :         r = find_rel(rels, je->r);
     572      271988 :         if (l && r) {
     573             :                 list *res;
     574      271844 :                 list *n_rels = sa_list(sa);
     575             : 
     576      271844 :                 append(n_rels, l);
     577      271844 :                 append(n_rels, r);
     578      271844 :                 res = list_select(exps, n_rels, (fcmp) &exp_joins_rels, (fdup)NULL);
     579      271844 :                 return res;
     580             :         }
     581         144 :         return sa_list(sa);
     582             : }
     583             : 
     584             : static int
     585        6024 : sql_column_kc_cmp(sql_column *c, sql_kc *kc)
     586             : {
     587             :         /* return on equality */
     588        6024 :         return (c->colnr - kc->c->colnr);
     589             : }
     590             : 
     591             : static sql_idx *
     592      360369 : find_fk_index(mvc *sql, sql_table *l, list *lcols, sql_table *r, list *rcols)
     593             : {
     594      360369 :         sql_trans *tr = sql->session->tr;
     595             : 
     596      360369 :         if (l->idxs) {
     597             :                 node *in;
     598      470004 :                 for (in = ol_first_node(l->idxs); in; in = in->next){
     599      110425 :                         sql_idx *li = in->data;
     600      110425 :                         if (li->type == join_idx) {
     601        7873 :                                 sql_key *rk = (sql_key*)os_find_id(tr->cat->objects, tr, ((sql_fkey*)li->key)->rkey);
     602             :                                 fcmp cmp = (fcmp)&sql_column_kc_cmp;
     603             : 
     604        8765 :                                 if (rk->t == r &&
     605        1682 :                                         list_match(lcols, li->columns, cmp) == 0 &&
     606         790 :                                         list_match(rcols, rk->columns, cmp) == 0) {
     607         790 :                                         return li;
     608             :                                 }
     609             :                         }
     610             :                 }
     611             :         }
     612             :         return NULL;
     613             : }
     614             : 
     615             : static sql_rel *
     616      543976 : find_basetable( sql_rel *r)
     617             : {
     618      779284 :         if (!r)
     619             :                 return NULL;
     620      774279 :         switch(r->op) {
     621      467144 :         case op_basetable:
     622      467144 :                 if (!r->l)
     623           0 :                         return NULL;
     624             :                 return r;
     625      235308 :         case op_project:
     626             :         case op_select:
     627      235308 :                 return find_basetable(r->l);
     628             :         default:
     629             :                 return NULL;
     630             :         }
     631             : }
     632             : 
     633             : static int
     634      120073 : exps_count(list *exps)
     635             : {
     636             :         node *n;
     637      120073 :         int cnt = 0;
     638             : 
     639      120073 :         if (!exps)
     640             :                 return 0;
     641      248607 :         for (n = exps->h; n; n=n->next)
     642      128534 :                 exp_count(&cnt, n->data);
     643      120073 :         return cnt;
     644             : }
     645             : 
     646             : static list *
     647      239892 : order_join_expressions(mvc *sql, list *dje, list *rels)
     648             : {
     649      239892 :         list *res = sa_list(sql->sa);
     650             :         node *n = NULL;
     651      239892 :         int i, *keys, cnt = list_length(dje);
     652             :         void **data;
     653             : 
     654      239892 :         if (cnt == 0)
     655             :                 return res;
     656             : 
     657      239588 :         keys = SA_NEW_ARRAY(sql->ta, int, cnt);
     658      239588 :         data = SA_NEW_ARRAY(sql->ta, void*, cnt);
     659             : 
     660     1114449 :         for (n = dje->h, i = 0; n; n = n->next, i++) {
     661      874861 :                 sql_exp *e = n->data;
     662             : 
     663      874861 :                 keys[i] = exp_keyvalue(e);
     664             :                 /* add some weight for the selections */
     665      874861 :                 if (e->type == e_cmp && !is_complex_exp(e->flag)) {
     666      874783 :                         sql_rel *l = find_rel(rels, e->l);
     667      874783 :                         sql_rel *r = find_rel(rels, e->r);
     668             : 
     669      874783 :                         if (l && is_select(l->op) && l->exps)
     670       75902 :                                 keys[i] += list_length(l->exps)*10 + exps_count(l->exps);
     671      874783 :                         if (r && is_select(r->op) && r->exps)
     672       44171 :                                 keys[i] += list_length(r->exps)*10 + exps_count(r->exps);
     673             :                 }
     674      874861 :                 data[i] = n->data;
     675             :         }
     676             :         /* sort descending */
     677      239588 :         GDKqsort(keys, data, NULL, cnt, sizeof(int), sizeof(void *), TYPE_int, true, true);
     678     1114449 :         for(i=0; i<cnt; i++) {
     679      874861 :                 list_append(res, data[i]);
     680             :         }
     681             :         return res;
     682             : }
     683             : 
     684             : static int
     685      186757 : find_join_rels(list **L, list **R, list *exps, list *rels)
     686             : {
     687             :         node *n;
     688             : 
     689      186757 :         *L = sa_list(exps->sa);
     690      186757 :         *R = sa_list(exps->sa);
     691      186757 :         if (!exps || list_length(exps) <= 1)
     692             :                 return -1;
     693      191667 :         for(n = exps->h; n; n = n->next) {
     694      145145 :                 sql_exp *e = n->data;
     695             :                 sql_rel *l = NULL, *r = NULL;
     696             : 
     697      145145 :                 if (!is_complex_exp(e->flag)){
     698      145139 :                         l = find_rel(rels, e->l);
     699      145139 :                         r = find_rel(rels, e->r);
     700             :                 }
     701      145145 :                 if (l<r) {
     702       78890 :                         list_append(*L, l);
     703       78890 :                         list_append(*R, r);
     704             :                 } else {
     705       66255 :                         list_append(*L, r);
     706       66255 :                         list_append(*R, l);
     707             :                 }
     708             :         }
     709             :         return 0;
     710             : }
     711             : 
     712             : static list *
     713       46522 : distinct_join_exps(list *aje, list *lrels, list *rrels)
     714             : {
     715             :         node *n, *m, *o, *p;
     716       46522 :         int len = list_length(aje), i, j;
     717       46522 :         char *used = SA_ZNEW_ARRAY(aje->sa, char, len);
     718       46522 :         list *res = sa_list(aje->sa);
     719             : 
     720       46522 :         assert(len == list_length(lrels));
     721      191667 :         for(n = lrels->h, m = rrels->h, j = 0; n && m;
     722      145145 :             n = n->next, m = m->next, j++) {
     723      145145 :                 if (n->data && m->data)
     724      791174 :                 for(o = n->next, p = m->next, i = j+1; o && p;
     725      646098 :                     o = o->next, p = p->next, i++) {
     726      646098 :                         if (o->data == n->data && p->data == m->data)
     727       12791 :                                 used[i] = 1;
     728             :                 }
     729             :         }
     730      191667 :         for (i = 0, n = aje->h; i < len; n = n->next, i++) {
     731      145145 :                 if (!used[i])
     732      135711 :                         list_append(res, n->data);
     733             :         }
     734       46522 :         return res;
     735             : }
     736             : 
     737             : static list *
     738      186757 : find_fk( mvc *sql, list *rels, list *exps)
     739             : {
     740             :         node *djn;
     741             :         list *sdje, *aje, *dje;
     742             :         list *lrels, *rrels;
     743             : 
     744             :         /* first find the distinct join expressions */
     745      186757 :         aje = list_select(exps, rels, (fcmp) &exp_is_join, (fdup)NULL);
     746             :         /* add left/right relation */
     747      186757 :         if (find_join_rels(&lrels, &rrels, aje, rels) < 0)
     748             :                 dje = aje;
     749             :         else
     750       46522 :                 dje = distinct_join_exps(aje, lrels, rrels);
     751      462340 :         for(djn=dje->h; djn; djn = djn->next) {
     752             :                 /* equal join expressions */
     753             :                 sql_idx *idx = NULL;
     754      275638 :                 sql_exp *je = djn->data, *le = je->l, *re = je->r;
     755             : 
     756      275638 :                 if (is_complex_exp(je->flag))
     757             :                         break;
     758      275583 :                 if (!find_prop(je->p, PROP_JOINIDX)) {
     759             :                         int swapped = 0;
     760      271988 :                         list *aaje = matching_joins(sql->sa, rels, aje, je);
     761      271988 :                         list *eje = list_select(aaje, (void*)1, (fcmp) &exp_is_eqjoin, (fdup)NULL);
     762      271988 :                         sql_rel *lr = find_rel(rels, le), *olr = lr;
     763      271988 :                         sql_rel *rr = find_rel(rels, re), *orr = rr;
     764      271988 :                         sql_rel *bt = NULL;
     765             :                         char *iname;
     766             : 
     767             :                         sql_table *l, *r;
     768      271988 :                         list *lexps = list_map(eje, lr, (fmap) &joinexp_col);
     769      271988 :                         list *rexps = list_map(eje, rr, (fmap) &joinexp_col);
     770             :                         list *lcols, *rcols;
     771             : 
     772      271988 :                         lr = find_basetable(lr);
     773      271988 :                         rr = find_basetable(rr);
     774      271988 :                         if (!lr || !rr)
     775       91470 :                                 continue;
     776      205997 :                         l = lr->l;
     777      205997 :                         r = rr->l;
     778      205997 :                         lcols = list_map(lexps, lr, (fmap) &table_colexp);
     779      205997 :                         rcols = list_map(rexps, rr, (fmap) &table_colexp);
     780      205997 :                         lcols->destroy = NULL;
     781      205997 :                         rcols->destroy = NULL;
     782      205997 :                         if (list_length(lcols) != list_length(rcols))
     783       25479 :                                 continue;
     784             : 
     785      180518 :                         idx = find_fk_index(sql, l, lcols, r, rcols);
     786      180518 :                         if (!idx) {
     787      179851 :                                 idx = find_fk_index(sql, r, rcols, l, lcols);
     788             :                                 swapped = 1;
     789             :                         }
     790             : 
     791      180518 :                         if (idx && (iname = sa_strconcat( sql->sa, "%", idx->base.name)) != NULL &&
     792         790 :                                    ((!swapped && name_find_column(olr, NULL, iname, -2, &bt) == NULL) ||
     793         123 :                                     ( swapped && name_find_column(orr, NULL, iname, -2, &bt) == NULL)))
     794             :                                 idx = NULL;
     795             : 
     796      180496 :                         if (idx) {
     797             :                                 prop *p;
     798             :                                 node *n;
     799             :                                 sql_exp *t = NULL, *i = NULL;
     800             : 
     801         768 :                                 if (list_length(lcols) > 1 || !mvc_debug_on(sql, 512)) {
     802             : 
     803             :                                         /* Add join between idx and TID */
     804         768 :                                         if (swapped) {
     805         113 :                                                 sql_exp *s = je->l, *l = je->r;
     806             : 
     807         113 :                                                 t = rel_find_column(sql->sa, olr, s->l, TID);
     808         113 :                                                 i = rel_find_column(sql->sa, orr, l->l, iname);
     809         113 :                                                 if (!t || !i)
     810           0 :                                                         continue;
     811         113 :                                                 je = exp_compare(sql->sa, i, t, cmp_equal);
     812             :                                         } else {
     813         655 :                                                 sql_exp *s = je->r, *l = je->l;
     814             : 
     815         655 :                                                 t = rel_find_column(sql->sa, orr, s->l, TID);
     816         655 :                                                 i = rel_find_column(sql->sa, olr, l->l, iname);
     817         655 :                                                 if (!t || !i)
     818           0 :                                                         continue;
     819         655 :                                                 je = exp_compare(sql->sa, i, t, cmp_equal);
     820             :                                         }
     821             : 
     822             :                                         /* Remove all join expressions */
     823        1538 :                                         for (n = eje->h; n; n = n->next)
     824         770 :                                                 list_remove_data(exps, NULL, n->data);
     825         768 :                                         append(exps, je);
     826         768 :                                         djn->data = je;
     827           0 :                                 } else if (swapped) { /* else keep je for single column expressions */
     828           0 :                                         je = exp_compare(sql->sa, je->r, je->l, cmp_equal);
     829             :                                         /* Remove all join expressions */
     830           0 :                                         for (n = eje->h; n; n = n->next)
     831           0 :                                                 list_remove_data(exps, NULL, n->data);
     832           0 :                                         append(exps, je);
     833           0 :                                         djn->data = je;
     834             :                                 }
     835         768 :                                 je->p = p = prop_create(sql->sa, PROP_JOINIDX, je->p);
     836         768 :                                 p->value = idx;
     837             :                         }
     838             :                 }
     839             :         }
     840             : 
     841             :         /* sort expressions on weighted number of reducing operators */
     842      186757 :         sdje = order_join_expressions(sql, dje, rels);
     843      186757 :         return sdje;
     844             : }
     845             : 
     846             : static sql_rel *
     847      126985 : order_joins(visitor *v, list *rels, list *exps)
     848             : {
     849             :         sql_rel *top = NULL, *l = NULL, *r = NULL;
     850             :         sql_exp *cje;
     851             :         node *djn;
     852      126985 :         list *sdje, *n_rels = sa_list(v->sql->sa);
     853             :         int fnd = 0;
     854             :         unsigned int rsingle;
     855             : 
     856             :         /* find foreign keys and reorder the expressions on reducing quality */
     857      126985 :         sdje = find_fk(v->sql, rels, exps);
     858             : 
     859      126985 :         if (list_length(rels) > 2 && mvc_debug_on(v->sql, 256)) {
     860           0 :                 for(djn = sdje->h; djn; djn = djn->next ) {
     861           0 :                         sql_exp *e = djn->data;
     862           0 :                         list_remove_data(exps, NULL, e);
     863             :                 }
     864           0 :                 top =  rel_planner(v->sql, rels, sdje, exps);
     865           0 :                 return top;
     866             :         }
     867             : 
     868             :         /* open problem, some expressions use more than 2 relations */
     869             :         /* For example a.x = b.y * c.z; */
     870      126985 :         if (list_length(rels) >= 2 && sdje->h) {
     871             :                 /* get the first expression */
     872      126795 :                 cje = sdje->h->data;
     873             : 
     874             :                 /* find the involved relations */
     875             : 
     876             :                 /* complex expressions may touch multiple base tables
     877             :                  * Should be pushed up to extra selection.
     878             :                  * */
     879      126795 :                 if (cje->type != e_cmp || is_complex_exp(cje->flag) || !find_prop(cje->p, PROP_HASHCOL) ||
     880          23 :                    (cje->type == e_cmp && cje->f == NULL)) {
     881      126795 :                         l = find_one_rel(rels, cje->l);
     882      126795 :                         r = find_one_rel(rels, cje->r);
     883             :                 }
     884             : 
     885      126795 :                 if (l && r && l != r) {
     886      126679 :                         list_remove_data(sdje, NULL, cje);
     887      126679 :                         list_remove_data(exps, NULL, cje);
     888             :                 }
     889             :         }
     890      126985 :         if (l && r && l != r) {
     891      126679 :                 list_remove_data(rels, NULL, l);
     892      126679 :                 list_remove_data(rels, NULL, r);
     893      126679 :                 list_append(n_rels, l);
     894      126679 :                 list_append(n_rels, r);
     895             : 
     896             :                 /* Create a relation between l and r. Since the calling
     897             :                    functions rewrote the join tree, into a list of expressions
     898             :                    and a list of (simple) relations, there are no outer joins
     899             :                    involved, we can simply do a crossproduct here.
     900             :                  */
     901      126679 :                 rsingle = is_single(r);
     902      126679 :                 reset_single(r);
     903      126679 :                 top = rel_crossproduct(v->sql->sa, l, r, op_join);
     904      126679 :                 if (rsingle)
     905           2 :                         set_single(r);
     906      126679 :                 rel_join_add_exp(v->sql->sa, top, cje);
     907             : 
     908             :                 /* all other join expressions on these 2 relations */
     909      217601 :                 for (node *en = exps->h; en; ) {
     910       90922 :                         node *next = en->next;
     911       90922 :                         sql_exp *e = en->data;
     912       90922 :                         if (rel_rebind_exp(v->sql, top, e)) {
     913        1707 :                                 rel_join_add_exp(v->sql->sa, top, e);
     914        1707 :                                 list_remove_data(exps, NULL, e);
     915             :                         }
     916             :                         en = next;
     917             :                 }
     918             :                 /* Remove other joins on the current 'n_rels' set in the distinct list too */
     919      215854 :                 for (node *en = sdje->h; en; ) {
     920       89175 :                         node *next = en->next;
     921       89175 :                         sql_exp *e = en->data;
     922       89175 :                         if (rel_rebind_exp(v->sql, top, e))
     923          52 :                                 list_remove_data(sdje, NULL, en->data);
     924             :                         en = next;
     925             :                 }
     926             :                 fnd = 1;
     927             :         }
     928             :         /* build join tree using the ordered list */
     929      212091 :         while(list_length(exps) && fnd) {
     930             :                 fnd = 0;
     931             :                 /* find the first expression which could be added */
     932       85106 :                 if (list_length(sdje) > 1)
     933       48658 :                         sdje = order_join_expressions(v->sql, sdje, rels);
     934      750264 :                 for(djn = sdje->h; djn && !fnd && rels->h; djn = (!fnd)?djn->next:NULL) {
     935             :                         node *ln, *rn, *en;
     936             : 
     937      332579 :                         cje = djn->data;
     938      332579 :                         ln = list_find(n_rels, cje->l, (fcmp)&rel_has_exp);
     939      332579 :                         rn = list_find(n_rels, cje->r, (fcmp)&rel_has_exp);
     940             : 
     941      332579 :                         if (ln && rn) {
     942           0 :                                 assert(0);
     943             :                                 /* create a selection on the current */
     944             :                                 l = ln->data;
     945             :                                 r = rn->data;
     946             :                                 rel_join_add_exp(v->sql->sa, top, cje);
     947             :                                 fnd = 1;
     948      332579 :                         } else if (ln || rn) {
     949       85059 :                                 if (ln) {
     950             :                                         l = ln->data;
     951       54383 :                                         r = find_rel(rels, cje->r);
     952             :                                 } else {
     953             :                                         l = rn->data;
     954       30676 :                                         r = find_rel(rels, cje->l);
     955             :                                 }
     956       85059 :                                 if (!r) {
     957             :                                         fnd = 1; /* not really, but this bails out */
     958           0 :                                         list_remove_data(sdje, NULL, cje); /* handle later as select */
     959           0 :                                         continue;
     960             :                                 }
     961             : 
     962             :                                 /* remove the expression from the lists */
     963       85059 :                                 list_remove_data(sdje, NULL, cje);
     964       85059 :                                 list_remove_data(exps, NULL, cje);
     965             : 
     966       85059 :                                 list_remove_data(rels, NULL, r);
     967       85059 :                                 append(n_rels, r);
     968             : 
     969             :                                 /* create a join using the current expression */
     970       85059 :                                 rsingle = is_single(r);
     971       85059 :                                 reset_single(r);
     972       85059 :                                 top = rel_crossproduct(v->sql->sa, top, r, op_join);
     973       85059 :                                 if (rsingle)
     974           0 :                                         set_single(r);
     975       85059 :                                 rel_join_add_exp(v->sql->sa, top, cje);
     976             : 
     977             :                                 /* all join expressions on these tables */
     978      626320 :                                 for (en = exps->h; en; ) {
     979      541261 :                                         node *next = en->next;
     980      541261 :                                         sql_exp *e = en->data;
     981      541261 :                                         if (rel_rebind_exp(v->sql, top, e)) {
     982        4109 :                                                 rel_join_add_exp(v->sql->sa, top, e);
     983        4109 :                                                 list_remove_data(exps, NULL, e);
     984             :                                         }
     985             :                                         en = next;
     986             :                                 }
     987             :                                 /* Remove other joins on the current 'n_rels'
     988             :                                    set in the distinct list too */
     989      626110 :                                 for (en = sdje->h; en; ) {
     990      541051 :                                         node *next = en->next;
     991      541051 :                                         sql_exp *e = en->data;
     992      541051 :                                         if (rel_rebind_exp(v->sql, top, e))
     993        4019 :                                                 list_remove_data(sdje, NULL, en->data);
     994             :                                         en = next;
     995             :                                 }
     996             :                                 fnd = 1;
     997             :                         }
     998             :                 }
     999             :         }
    1000      126985 :         if (list_length(rels)) { /* more relations */
    1001             :                 node *n;
    1002        2451 :                 for(n=rels->h; n; n = n->next) {
    1003        1632 :                         sql_rel *nr = n->data;
    1004             : 
    1005        1632 :                         if (top) {
    1006        1326 :                                 rsingle = is_single(nr);
    1007        1326 :                                 reset_single(nr);
    1008        1326 :                                 top = rel_crossproduct(v->sql->sa, top, nr, op_join);
    1009        1326 :                                 if (rsingle)
    1010          16 :                                         set_single(nr);
    1011             :                         } else
    1012             :                                 top = nr;
    1013             :                 }
    1014             :         }
    1015      126985 :         if (list_length(exps)) { /* more expressions (add selects) */
    1016         175 :                 top = rel_select(v->sql->sa, top, NULL);
    1017         361 :                 for(node *n=exps->h; n; n = n->next) {
    1018         186 :                         sql_exp *e = n->data;
    1019             : 
    1020             :                         /* find the involved relations */
    1021             : 
    1022             :                         /* complex expressions may touch multiple base tables
    1023             :                          * Should be push up to extra selection. */
    1024             :                         /*
    1025             :                         l = find_one_rel(rels, e->l);
    1026             :                         r = find_one_rel(rels, e->r);
    1027             : 
    1028             :                         if (l && r)
    1029             :                         */
    1030         186 :                         if (exp_is_join_exp(e) == 0) {
    1031             :                                 sql_rel *nr = NULL;
    1032         175 :                                 if (is_theta_exp(e->flag)) {
    1033         136 :                                         nr = rel_push_join(v->sql, top->l, e->l, e->r, e->f, e, 0);
    1034          39 :                                 } else if (e->flag == cmp_filter || e->flag == cmp_or) {
    1035          39 :                                         sql_exp *l = exps_find_one_multi_exp(e->l), *r = exps_find_one_multi_exp(e->r);
    1036          39 :                                         if (l && r)
    1037          36 :                                                 nr = rel_push_join(v->sql, top->l, l, r, NULL, e, 0);
    1038             :                                 }
    1039         172 :                                 if (!nr)
    1040           3 :                                         rel_join_add_exp(v->sql->sa, top->l, e);
    1041             :                         } else
    1042          11 :                                 rel_select_add_exp(v->sql->sa, top, e);
    1043             :                 }
    1044         175 :                 if (list_empty(top->exps)) { /* empty select */
    1045         164 :                         sql_rel *l = top->l;
    1046         164 :                         top->l = NULL;
    1047         164 :                         rel_destroy(top);
    1048             :                         top = l;
    1049             :                 }
    1050             :         }
    1051             :         return top;
    1052             : }
    1053             : 
    1054             : static int
    1055      340199 : rel_neg_in_size(sql_rel *r)
    1056             : {
    1057      340199 :         if (is_union(r->op) && r->nrcols == 0)
    1058           0 :                 return -1 + rel_neg_in_size(r->l);
    1059      340199 :         if (is_project(r->op) && r->nrcols == 0)
    1060         150 :                 return -1;
    1061             :         return 0;
    1062             : }
    1063             : 
    1064      340199 : static void _rel_destroy(void *dummy, sql_rel *rel)
    1065             : {
    1066             :         (void)dummy;
    1067      340199 :         rel_destroy(rel);
    1068      340199 : }
    1069             : 
    1070             : static list *
    1071      126985 : push_in_join_down(mvc *sql, list *rels, list *exps)
    1072             : {
    1073             :         node *n;
    1074             :         int restart = 1;
    1075             :         list *nrels;
    1076             : 
    1077             :         /* we should sort these first, ie small in's before large one's */
    1078      126985 :         nrels = list_sort(rels, (fkeyvalue)&rel_neg_in_size, (fdup)&rel_dup);
    1079             : 
    1080             :         /* we need to cleanup, the new refs ! */
    1081      126985 :         rels->destroy = (fdestroy)_rel_destroy;
    1082      126985 :         list_destroy(rels);
    1083             :         rels = nrels;
    1084             : 
    1085             :         /* one of the rels should be a op_union with nrcols == 0 */
    1086      253970 :         while (restart) {
    1087      467334 :                 for (n = rels->h; n; n = n->next) {
    1088      340349 :                         sql_rel *r = n->data;
    1089             : 
    1090             :                         restart = 0;
    1091      340349 :                         if (is_project(r->op) && r->nrcols == 0) {
    1092             :                                 /* next step find expression on this relation */
    1093             :                                 node *m;
    1094             :                                 sql_rel *l = NULL;
    1095             :                                 sql_exp *je = NULL;
    1096             : 
    1097         300 :                                 for(m = exps->h; !je && m; m = m->next) {
    1098         150 :                                         sql_exp *e = m->data;
    1099             : 
    1100         150 :                                         if (e->type == e_cmp && e->flag == cmp_equal) {
    1101             :                                                 /* in values are on
    1102             :                                                         the right of the join */
    1103         150 :                                                 if (rel_has_exp(r, e->r) >= 0)
    1104             :                                                         je = e;
    1105             :                                         }
    1106             :                                 }
    1107             :                                 /* with this expression find other relation */
    1108         150 :                                 if (je && (l = find_rel(rels, je->l)) != NULL) {
    1109         150 :                                         unsigned int rsingle = is_single(r);
    1110         150 :                                         reset_single(r);
    1111         150 :                                         sql_rel *nr = rel_crossproduct(sql->sa, l, r, op_join);
    1112         150 :                                         if (rsingle)
    1113           0 :                                                 set_single(r);
    1114         150 :                                         rel_join_add_exp(sql->sa, nr, je);
    1115         150 :                                         list_append(rels, nr);
    1116         150 :                                         list_remove_data(rels, NULL, l);
    1117         150 :                                         list_remove_data(rels, NULL, r);
    1118         150 :                                         list_remove_data(exps, NULL, je);
    1119             :                                         restart = 1;
    1120             :                                         break;
    1121             :                                 }
    1122             : 
    1123             :                         }
    1124             :                 }
    1125             :         }
    1126      126985 :         return rels;
    1127             : }
    1128             : 
    1129             : static list *
    1130      553495 : push_up_join_exps( mvc *sql, sql_rel *rel)
    1131             : {
    1132      553495 :         if (rel_is_ref(rel))
    1133             :                 return NULL;
    1134             : 
    1135      519630 :         switch(rel->op) {
    1136      213264 :         case op_join: {
    1137      213264 :                 sql_rel *rl = rel->l;
    1138      213264 :                 sql_rel *rr = rel->r;
    1139             :                 list *l, *r;
    1140             : 
    1141      213264 :                 if (rel_is_ref(rl) && rel_is_ref(rr)) {
    1142          34 :                         l = rel->exps;
    1143          34 :                         rel->exps = NULL;
    1144          34 :                         return l;
    1145             :                 }
    1146      213230 :                 l = push_up_join_exps(sql, rl);
    1147      213230 :                 r = push_up_join_exps(sql, rr);
    1148      213230 :                 if (l && r) {
    1149         241 :                         l = list_merge(l, r, (fdup)NULL);
    1150             :                         r = NULL;
    1151      212989 :                 } else if (!l) {
    1152             :                         l = r;
    1153             :                         r = NULL;
    1154             :                 }
    1155      213230 :                 if (rel->exps) {
    1156      195557 :                         if (l && !r)
    1157             :                                 r = l;
    1158      195557 :                         l = list_merge(rel->exps, r, (fdup)NULL);
    1159             :                 }
    1160      213230 :                 rel->exps = NULL;
    1161      213230 :                 return l;
    1162             :         }
    1163             :         default:
    1164             :                 return NULL;
    1165             :         }
    1166             : }
    1167             : 
    1168             : static sql_rel *
    1169      181787 : reorder_join(visitor *v, sql_rel *rel)
    1170             : {
    1171             :         list *exps;
    1172             :         list *rels;
    1173             : 
    1174      181787 :         if (rel->op == op_join && !rel_is_ref(rel))
    1175      127035 :                 rel->exps = push_up_join_exps(v->sql, rel);
    1176             : 
    1177      181787 :         exps = rel->exps;
    1178      181787 :         if (!exps) /* crosstable, ie order not important */
    1179             :                 return rel;
    1180      181043 :         rel->exps = NULL; /* should be all crosstables by now */
    1181      181043 :         rels = sa_list(v->sql->sa);
    1182      181043 :         if (is_outerjoin(rel->op) || is_single(rel)) {
    1183             :                 sql_rel *l, *r;
    1184             :                 int cnt = 0;
    1185             :                 /* try to use an join index also for outer joins */
    1186       53576 :                 get_inner_relations(v->sql, rel, rels);
    1187       53576 :                 cnt = list_length(exps);
    1188       53576 :                 rel->exps = find_fk(v->sql, rels, exps);
    1189       53576 :                 if (list_length(rel->exps) != cnt)
    1190        4477 :                         rel->exps = order_join_expressions(v->sql, exps, rels);
    1191       53576 :                 l = rel->l;
    1192       53576 :                 r = rel->r;
    1193       53576 :                 if (is_join(l->op))
    1194       26669 :                         rel->l = reorder_join(v, rel->l);
    1195       53576 :                 if (is_join(r->op))
    1196         107 :                         rel->r = reorder_join(v, rel->r);
    1197             :         } else {
    1198      127467 :                 get_relations(v, rel, rels);
    1199      127467 :                 if (list_length(rels) > 1) {
    1200      126985 :                         rels = push_in_join_down(v->sql, rels, exps);
    1201      126985 :                         rel = order_joins(v, rels, exps);
    1202             :                 } else {
    1203         482 :                         rel->exps = exps;
    1204             :                         exps = NULL;
    1205             :                 }
    1206             :         }
    1207             :         return rel;
    1208             : }
    1209             : 
    1210             : static sql_rel *
    1211     1680286 : rel_join_order(visitor *v, sql_rel *rel)
    1212             : {
    1213     1680286 :         if (!rel)
    1214             :                 return rel;
    1215             : 
    1216     1673072 :         switch (rel->op) {
    1217             :         case op_basetable:
    1218             :         case op_table:
    1219             :                 break;
    1220             :         case op_join:
    1221             :         case op_left:
    1222             :         case op_right:
    1223             :         case op_full:
    1224             :                 break;
    1225             : 
    1226      173792 :         case op_semi:
    1227             :         case op_anti:
    1228             : 
    1229             :         case op_union:
    1230             :         case op_inter:
    1231             :         case op_except:
    1232             :         case op_merge:
    1233      173792 :                 rel->l = rel_join_order(v, rel->l);
    1234      173792 :                 rel->r = rel_join_order(v, rel->r);
    1235      173792 :                 break;
    1236      848066 :         case op_project:
    1237             :         case op_select:
    1238             :         case op_groupby:
    1239             :         case op_topn:
    1240             :         case op_sample:
    1241      848066 :                 rel->l = rel_join_order(v, rel->l);
    1242      848066 :                 break;
    1243        3408 :         case op_ddl:
    1244        3408 :                 if (rel->flag == ddl_output || rel->flag == ddl_create_seq || rel->flag == ddl_alter_seq || rel->flag == ddl_alter_table || rel->flag == ddl_create_table || rel->flag == ddl_create_view) {
    1245           0 :                         rel->l = rel_join_order(v, rel->l);
    1246        3408 :                 } else if (rel->flag == ddl_list || rel->flag == ddl_exception) {
    1247          46 :                         rel->l = rel_join_order(v, rel->l);
    1248          46 :                         rel->r = rel_join_order(v, rel->r);
    1249             :                 }
    1250             :                 break;
    1251        2992 :         case op_insert:
    1252             :         case op_update:
    1253             :         case op_delete:
    1254        2992 :                 rel->r = rel_join_order(v, rel->r);
    1255        2992 :                 break;
    1256             :         case op_truncate:
    1257             :                 break;
    1258             :         }
    1259     1673072 :         if (is_join(rel->op) && rel->exps && !rel_is_ref(rel)) {
    1260             :                 if (rel && !rel_is_ref(rel))
    1261      155011 :                         rel = reorder_join(v, rel);
    1262     1518061 :         } else if (is_join(rel->op)) {
    1263       24543 :                 rel->l = rel_join_order(v, rel->l);
    1264       24543 :                 rel->r = rel_join_order(v, rel->r);
    1265             :         }
    1266             :         return rel;
    1267             : }
    1268             : 
    1269             : /* exp_rename */
    1270             : static sql_exp * exp_rename(mvc *sql, sql_exp *e, sql_rel *f, sql_rel *t);
    1271             : 
    1272             : static list *
    1273       37020 : exps_rename(mvc *sql, list *l, sql_rel *f, sql_rel *t)
    1274             : {
    1275       37020 :         if (list_empty(l))
    1276             :                 return l;
    1277       21713 :         for (node *n=l->h; n; n=n->next)
    1278       14789 :                 n->data = exp_rename(sql, n->data, f, t);
    1279             :         return l;
    1280             : }
    1281             : 
    1282             : /* exp_rename */
    1283             : static sql_exp *
    1284      186190 : exp_rename(mvc *sql, sql_exp *e, sql_rel *f, sql_rel *t)
    1285             : {
    1286             :         sql_exp *ne = NULL;
    1287             : 
    1288      186190 :         switch(e->type) {
    1289       80763 :         case e_column:
    1290       80763 :                 if (e->l) {
    1291       80740 :                         ne = exps_bind_column2(f->exps, e->l, e->r, NULL);
    1292             :                         /* if relation name matches expressions relation name, find column based on column name alone */
    1293             :                 } else {
    1294          23 :                         ne = exps_bind_column(f->exps, e->r, NULL, NULL, 1);
    1295             :                 }
    1296       80763 :                 if (!ne)
    1297             :                         return e;
    1298             :                 e = NULL;
    1299       52619 :                 if (exp_name(ne) && ne->r && ne->l)
    1300       51844 :                         e = rel_bind_column2(sql, t, ne->l, ne->r, 0);
    1301       52619 :                 if (!e && ne->r)
    1302         706 :                         e = rel_bind_column(sql, t, ne->r, 0, 1);
    1303       52619 :                 if (!e) {
    1304          69 :                         sql->session->status = 0;
    1305          69 :                         sql->errstr[0] = 0;
    1306          69 :                         if (exp_is_atom(ne))
    1307             :                                 return ne;
    1308             :                 }
    1309       52550 :                 return exp_ref(sql, e);
    1310       51900 :         case e_cmp:
    1311       51900 :                 if (e->flag == cmp_or || e->flag == cmp_filter) {
    1312         370 :                         e->l = exps_rename(sql, e->l, f, t);
    1313         370 :                         e->r = exps_rename(sql, e->r, f, t);
    1314       51530 :                 } else if (e->flag == cmp_in || e->flag == cmp_notin) {
    1315        4613 :                         e->l = exp_rename(sql, e->l, f, t);
    1316        4613 :                         e->r = exps_rename(sql, e->r, f, t);
    1317             :                 } else {
    1318       46917 :                         e->l = exp_rename(sql, e->l, f, t);
    1319       46917 :                         e->r = exp_rename(sql, e->r, f, t);
    1320       46917 :                         if (e->f)
    1321         107 :                                 e->f = exp_rename(sql, e->f, f, t);
    1322             :                 }
    1323             :                 break;
    1324       21860 :         case e_convert:
    1325       21860 :                 e->l = exp_rename(sql, e->l, f, t);
    1326       21860 :                 break;
    1327        1571 :         case e_aggr:
    1328             :         case e_func:
    1329        1571 :                 e->l = exps_rename(sql, e->l, f, t);
    1330        1571 :                 break;
    1331       30096 :         case e_atom:
    1332       30096 :                 e->f = exps_rename(sql, e->f, f, t);
    1333       30096 :                 break;
    1334             :         case e_psm:
    1335             :                 break;
    1336             :         }
    1337             :         return e;
    1338             : }
    1339             : 
    1340             : static int
    1341     2039461 : can_push_func(sql_exp *e, sql_rel *rel, int *must, int depth)
    1342             : {
    1343     2137294 :         switch(e->type) {
    1344     1786126 :         case e_cmp: {
    1345     1786126 :                 sql_exp *l = e->l, *r = e->r, *f = e->f;
    1346             : 
    1347     1786126 :                 if (e->flag == cmp_or || e->flag == cmp_in || e->flag == cmp_notin || e->flag == cmp_filter)
    1348             :                         return 0;
    1349     1685480 :                 if (depth > 0) { /* for comparisons under the top ones, they become functions */
    1350          35 :                         int lmust = 0;
    1351          35 :                         int res = can_push_func(l, rel, &lmust, depth + 1) && can_push_func(r, rel, &lmust, depth + 1) &&
    1352          10 :                                         (!f || can_push_func(f, rel, &lmust, depth + 1));
    1353           8 :                         if (res && !lmust)
    1354             :                                 return 1;
    1355          35 :                         (*must) |= lmust;
    1356          35 :                         return res;
    1357             :                 } else {
    1358     1685445 :                         int mustl = 0, mustr = 0, mustf = 0;
    1359     1685445 :                         return ((l->type == e_column || can_push_func(l, rel, &mustl, depth + 1)) && (*must = mustl)) ||
    1360     3356810 :                                         ((r->type == e_column || can_push_func(r, rel, &mustr, depth + 1)) && (*must = mustr)) ||
    1361        2991 :                                         ((f && (f->type == e_column || can_push_func(f, rel, &mustf, depth + 1)) && (*must = mustf)));
    1362             :                 }
    1363             :         }
    1364       97833 :         case e_convert:
    1365       97833 :                 return can_push_func(e->l, rel, must, depth + 1);
    1366        9475 :         case e_aggr:
    1367             :         case e_func: {
    1368        9475 :                 list *l = e->l;
    1369        9475 :                 int res = 1, lmust = 0;
    1370             : 
    1371        9475 :                 if (exp_unsafe(e, 0))
    1372             :                         return 0;
    1373       24893 :                 if (l) for (node *n = l->h; n && res; n = n->next)
    1374       15418 :                         res &= can_push_func(n->data, rel, &lmust, depth + 1);
    1375        9475 :                 if (res && !lmust)
    1376             :                         return 1;
    1377        8154 :                 (*must) |= lmust;
    1378        8154 :                 return res;
    1379             :         }
    1380      100667 :         case e_column:
    1381      100667 :                 if (rel && !rel_find_exp(rel, e))
    1382             :                         return 0;
    1383       60187 :                 (*must) = 1;
    1384             :                 /* fall through */
    1385             :         default:
    1386             :                 return 1;
    1387             :         }
    1388             : }
    1389             : 
    1390             : static int
    1391     1255374 : exps_can_push_func(list *exps, sql_rel *rel)
    1392             : {
    1393     5325218 :         for(node *n = exps->h; n; n = n->next) {
    1394     4124249 :                 sql_exp *e = n->data;
    1395     4124249 :                 int mustl = 0, mustr = 0;
    1396             : 
    1397     4124249 :                 if ((is_joinop(rel->op) || is_select(rel->op)) && ((can_push_func(e, rel->l, &mustl, 0) && mustl)))
    1398       54405 :                         return 1;
    1399     4109831 :                 if (is_joinop(rel->op) && can_push_func(e, rel->r, &mustr, 0) && mustr)
    1400             :                         return 1;
    1401             :         }
    1402             :         return 0;
    1403             : }
    1404             : 
    1405             : static int
    1406      205159 : exp_needs_push_down(sql_exp *e)
    1407             : {
    1408      260432 :         switch(e->type) {
    1409       68475 :         case e_cmp:
    1410       68475 :                 if (e->flag == cmp_or || e->flag == cmp_in || e->flag == cmp_notin || e->flag == cmp_filter)
    1411             :                         return 0;
    1412       67789 :                 return exp_needs_push_down(e->l) || exp_needs_push_down(e->r) || (e->f && exp_needs_push_down(e->f));
    1413       55273 :         case e_convert:
    1414       55273 :                 return exp_needs_push_down(e->l);
    1415        2774 :         case e_aggr:
    1416             :         case e_func:
    1417        2774 :                 if (!e->l || exps_are_atoms(e->l))
    1418          48 :                         return 0;
    1419             :                 return 1;
    1420       19892 :         case e_atom:
    1421       19892 :                 if (!e->f || exps_are_atoms(e->f))
    1422       19892 :                         return 0;
    1423             :                 return 1;
    1424             :         case e_column:
    1425             :         default:
    1426             :                 return 0;
    1427             :         }
    1428             : }
    1429             : 
    1430             : static int
    1431       54405 : exps_need_push_down( list *exps )
    1432             : {
    1433      120146 :         for(node *n = exps->h; n; n = n->next)
    1434       68467 :                 if (exp_needs_push_down(n->data))
    1435             :                         return 1;
    1436             :         return 0;
    1437             : }
    1438             : 
    1439             : static sql_exp *exp_push_single_func_down(visitor *v, sql_rel *rel, sql_rel *ol, sql_rel *or, sql_exp *e, int depth);
    1440             : 
    1441             : static list *
    1442        3219 : exps_push_single_func_down(visitor *v, sql_rel *rel, sql_rel *ol, sql_rel *or, list *exps, int depth)
    1443             : {
    1444        3219 :         if (mvc_highwater(v->sql))
    1445           0 :                 return sql_error(v->sql, 10, SQLSTATE(42000) "Query too complex: running out of stack space");
    1446             : 
    1447        9755 :         for (node *n = exps->h; n; n = n->next)
    1448        6536 :                 if ((n->data = exp_push_single_func_down(v, rel, ol, or, n->data, depth)) == NULL)
    1449             :                         return NULL;
    1450             :         return exps;
    1451             : }
    1452             : 
    1453             : static sql_exp *
    1454       22620 : exp_push_single_func_down(visitor *v, sql_rel *rel, sql_rel *ol, sql_rel *or, sql_exp *e, int depth)
    1455             : {
    1456       22620 :         if (mvc_highwater(v->sql))
    1457           0 :                 return sql_error(v->sql, 10, SQLSTATE(42000) "Query too complex: running out of stack space");
    1458             : 
    1459       22620 :         switch(e->type) {
    1460        6512 :         case e_cmp: {
    1461        6512 :                 if (e->flag == cmp_or || e->flag == cmp_filter) {
    1462         242 :                         if ((e->l = exps_push_single_func_down(v, rel, ol, or, e->l, depth + 1)) == NULL)
    1463             :                                 return NULL;
    1464         242 :                         if ((e->r = exps_push_single_func_down(v, rel, ol, or, e->r, depth + 1)) == NULL)
    1465           0 :                                 return NULL;
    1466        6270 :                 } else if (e->flag == cmp_in || e->flag == cmp_notin) {
    1467           9 :                         if ((e->l = exp_push_single_func_down(v, rel, ol, or, e->l, depth + 1)) == NULL)
    1468             :                                 return NULL;
    1469           9 :                         if ((e->r = exps_push_single_func_down(v, rel, ol, or, e->r, depth + 1)) == NULL)
    1470           0 :                                 return NULL;
    1471             :                 } else {
    1472        6261 :                         if ((e->l = exp_push_single_func_down(v, rel, ol, or, e->l, depth + 1)) == NULL)
    1473             :                                 return NULL;
    1474        6261 :                         if ((e->r = exp_push_single_func_down(v, rel, ol, or, e->r, depth + 1)) == NULL)
    1475             :                                 return NULL;
    1476        6261 :                         if (e->f && (e->f = exp_push_single_func_down(v, rel, ol, or, e->f, depth + 1)) == NULL)
    1477           0 :                                 return NULL;
    1478             :                 }
    1479             :         } break;
    1480        2119 :         case e_convert:
    1481        2119 :                 if ((e->l = exp_push_single_func_down(v, rel, ol, or, e->l, depth + 1)) == NULL)
    1482           0 :                         return NULL;
    1483             :                 break;
    1484        4566 :         case e_aggr:
    1485             :         case e_func: {
    1486        4566 :                 sql_rel *l = rel->l, *r = rel->r;
    1487        4566 :                 int must = 0, mustl = 0, mustr = 0;
    1488             : 
    1489        4566 :                 if (exp_unsafe(e, 0))
    1490          17 :                         return e;
    1491        4566 :                 if (!e->l || exps_are_atoms(e->l))
    1492          17 :                         return e;
    1493        4549 :                 if ((is_joinop(rel->op) && ((can_push_func(e, l, &mustl, depth + 1) && mustl) || (can_push_func(e, r, &mustr, depth + 1) && mustr))) ||
    1494        4111 :                         (is_select(rel->op) && can_push_func(e, l, &must, depth + 1) && must)) {
    1495        4530 :                         exp_label(v->sql->sa, e, ++v->sql->label);
    1496             :                         /* we need a full projection, group by's and unions cannot be extended with more expressions */
    1497        4530 :                         if (mustr) {
    1498          95 :                                 if (r == or) /* don't project twice */
    1499          73 :                                         rel->r = r = rel_project(v->sql->sa, r, rel_projections(v->sql, r, NULL, 1, 1));
    1500          95 :                                 list_append(r->exps, e);
    1501             :                         } else {
    1502        4435 :                                 if (l == ol) /* don't project twice */
    1503        2658 :                                         rel->l = l = rel_project(v->sql->sa, l, rel_projections(v->sql, l, NULL, 1, 1));
    1504        4435 :                                 list_append(l->exps, e);
    1505             :                         }
    1506        4530 :                         e = exp_ref(v->sql, e);
    1507        4530 :                         v->changes++;
    1508             :                 }
    1509        4549 :         } break;
    1510        3638 :         case e_atom: {
    1511        3638 :                 if (e->f && (e->f = exps_push_single_func_down(v, rel, ol, or, e->f, depth + 1)) == NULL)
    1512           0 :                         return NULL;
    1513             :         } break;
    1514             :         case e_column:
    1515             :         case e_psm:
    1516             :                 break;
    1517             :         }
    1518             :         return e;
    1519             : }
    1520             : 
    1521             : static inline sql_rel *
    1522     4827146 : rel_push_func_down(visitor *v, sql_rel *rel)
    1523             : {
    1524     4827146 :         if ((is_select(rel->op) || is_joinop(rel->op)) && rel->l && rel->exps && !(rel_is_ref(rel))) {
    1525     1358580 :                 int changes = v->changes;
    1526     1358580 :                 sql_rel *l = rel->l, *r = rel->r;
    1527             : 
    1528             :                 /* only push down when is useful */
    1529     1358580 :                 if ((is_select(rel->op) && list_length(rel->exps) <= 1) || rel_is_ref(l) || (is_joinop(rel->op) && rel_is_ref(r)))
    1530      567096 :                         return rel;
    1531      791484 :                 if (exps_can_push_func(rel->exps, rel) && exps_need_push_down(rel->exps) && !exps_push_single_func_down(v, rel, l, r, rel->exps, 0))
    1532             :                         return NULL;
    1533      791484 :                 if (v->changes > changes) /* once we get a better join order, we can try to remove this projection */
    1534        2720 :                         return rel_project(v->sql->sa, rel, rel_projections(v->sql, rel, NULL, 1, 1));
    1535             :         }
    1536     4257330 :         if (is_simple_project(rel->op) && rel->l && rel->exps) {
    1537             :                 sql_rel *pl = rel->l;
    1538             : 
    1539     1373549 :                 if (is_joinop(pl->op) && exps_can_push_func(rel->exps, rel)) {
    1540           0 :                         sql_rel *l = pl->l, *r = pl->r, *ol = l, *or = r;
    1541             : 
    1542           0 :                         for (node *n = rel->exps->h; n; ) {
    1543           0 :                                 node *next = n->next;
    1544           0 :                                 sql_exp *e = n->data;
    1545           0 :                                 int mustl = 0, mustr = 0;
    1546             : 
    1547           0 :                                 if ((can_push_func(e, l, &mustl, 0) && mustl) || (can_push_func(e, r, &mustr, 0) && mustr)) {
    1548           0 :                                         if (mustl) {
    1549           0 :                                                 if (l == ol) /* don't project twice */
    1550           0 :                                                         pl->l = l = rel_project(v->sql->sa, l, rel_projections(v->sql, l, NULL, 1, 1));
    1551           0 :                                                 list_append(l->exps, e);
    1552           0 :                                                 list_remove_node(rel->exps, NULL, n);
    1553           0 :                                                 v->changes++;
    1554             :                                         } else {
    1555           0 :                                                 if (r == or) /* don't project twice */
    1556           0 :                                                         pl->r = r = rel_project(v->sql->sa, r, rel_projections(v->sql, r, NULL, 1, 1));
    1557           0 :                                                 list_append(r->exps, e);
    1558           0 :                                                 list_remove_node(rel->exps, NULL, n);
    1559           0 :                                                 v->changes++;
    1560             :                                         }
    1561             :                                 }
    1562             :                                 n = next;
    1563             :                         }
    1564             :                 }
    1565             :         }
    1566             :         return rel;
    1567             : }
    1568             : 
    1569             : /*
    1570             :  * Push Count inside crossjoin down, and multiply the results
    1571             :  *
    1572             :  *     project (                                project(
    1573             :  *          group by (                               crossproduct (
    1574             :  *              crossproduct(                             project (
    1575             :  *                   L,                  =>                    group by (
    1576             :  *                   R                                              L
    1577             :  *              ) [ ] [ count NOT NULL ]                       ) [ ] [ count NOT NULL ]
    1578             :  *          )                                             ),
    1579             :  *     ) [ NOT NULL ]                                     project (
    1580             :  *                                                              group by (
    1581             :  *                                                                  R
    1582             :  *                                                              ) [ ] [ count NOT NULL ]
    1583             :  *                                                        )
    1584             :  *                                                   ) [ sql_mul(.., .. NOT NULL) ]
    1585             :  *                                              )
    1586             :  */
    1587             : static inline sql_rel *
    1588      200583 : rel_push_count_down(visitor *v, sql_rel *rel)
    1589             : {
    1590      200583 :         sql_rel *r = rel->l;
    1591             : 
    1592      200583 :         assert(is_groupby(rel->op));
    1593      200583 :         if (!rel_is_ref(rel) && list_empty(rel->r) &&
    1594       47591 :                 r && !r->exps && r->op == op_join && !(rel_is_ref(r)) &&
    1595             :                 /* currently only single count aggregation is handled, no other projects or aggregation */
    1596          72 :                 list_length(rel->exps) == 1 && exp_aggr_is_count(rel->exps->h->data)) {
    1597             :                 sql_exp *nce, *oce, *cnt1 = NULL, *cnt2 = NULL;
    1598             :                 sql_rel *gbl = NULL, *gbr = NULL;       /* Group By */
    1599             :                 sql_rel *cp = NULL;                                     /* Cross Product */
    1600             :                 sql_rel *srel;
    1601             : 
    1602          23 :                 oce = rel->exps->h->data;
    1603          23 :                 if (oce->l) /* we only handle COUNT(*) */
    1604             :                         return rel;
    1605             : 
    1606          21 :                 srel = r->l;
    1607             :                 {
    1608          21 :                         sql_subfunc *cf = sql_bind_func(v->sql, "sys", "count", sql_bind_localtype("void"), NULL, F_AGGR);
    1609          21 :                         sql_exp *e = exp_aggr(v->sql->sa, NULL, cf, need_distinct(oce), need_no_nil(oce), oce->card, 0);
    1610             : 
    1611          21 :                         exp_label(v->sql->sa, e, ++v->sql->label);
    1612          21 :                         cnt1 = exp_ref(v->sql, e);
    1613          21 :                         gbl = rel_groupby(v->sql, rel_dup(srel), NULL);
    1614          21 :                         set_processed(gbl);
    1615          21 :                         rel_groupby_add_aggr(v->sql, gbl, e);
    1616             :                 }
    1617             : 
    1618          21 :                 srel = r->r;
    1619             :                 {
    1620          21 :                         sql_subfunc *cf = sql_bind_func(v->sql, "sys", "count", sql_bind_localtype("void"), NULL, F_AGGR);
    1621          21 :                         sql_exp *e = exp_aggr(v->sql->sa, NULL, cf, need_distinct(oce), need_no_nil(oce), oce->card, 0);
    1622             : 
    1623          21 :                         exp_label(v->sql->sa, e, ++v->sql->label);
    1624          21 :                         cnt2 = exp_ref(v->sql, e);
    1625          21 :                         gbr = rel_groupby(v->sql, rel_dup(srel), NULL);
    1626          21 :                         set_processed(gbr);
    1627          21 :                         rel_groupby_add_aggr(v->sql, gbr, e);
    1628             :                 }
    1629             : 
    1630          21 :                 cp = rel_crossproduct(v->sql->sa, gbl, gbr, op_join);
    1631             : 
    1632          21 :                 if (!(nce = rel_binop_(v->sql, NULL, cnt1, cnt2, "sys", "sql_mul", card_value))) {
    1633           0 :                         v->sql->session->status = 0;
    1634           0 :                         v->sql->errstr[0] = '\0';
    1635           0 :                         return rel; /* error, fallback to original expression */
    1636             :                 }
    1637             :                 /* because of remote plans, make sure "sql_mul" returns bigint. The cardinality is atomic, so no major performance penalty */
    1638          21 :                 if (subtype_cmp(exp_subtype(oce), exp_subtype(nce)) != 0)
    1639          21 :                         nce = exp_convert(v->sql->sa, nce, exp_subtype(nce), exp_subtype(oce));
    1640          21 :                 if (exp_name(oce))
    1641          21 :                         exp_prop_alias(v->sql->sa, nce, oce);
    1642             : 
    1643          21 :                 rel_destroy(rel);
    1644          21 :                 rel = rel_project(v->sql->sa, cp, append(new_exp_list(v->sql->sa), nce));
    1645          21 :                 set_processed(rel);
    1646             : 
    1647          21 :                 v->changes++;
    1648             :         }
    1649             : 
    1650             :         return rel;
    1651             : }
    1652             : 
    1653             : static bool
    1654         856 : check_projection_on_foreignside(sql_rel *r, list *pexps, int fk_left)
    1655             : {
    1656             :         /* projection columns from the foreign side */
    1657         856 :         if (list_empty(pexps))
    1658             :                 return true;
    1659        2425 :         for (node *n = pexps->h; n; n = n->next) {
    1660        2353 :                 sql_exp *pe = n->data;
    1661             : 
    1662        2353 :                 if (pe && is_atom(pe->type))
    1663          25 :                         continue;
    1664        2328 :                 if (pe && !is_alias(pe->type))
    1665             :                         return false;
    1666             :                 /* check for columns from the pk side, then keep the join with the pk */
    1667        2316 :                 if ((fk_left && rel_find_exp(r->r, pe)) || (!fk_left && rel_find_exp(r->l, pe)))
    1668         721 :                         return false;
    1669             :         }
    1670             :         return true;
    1671             : }
    1672             : 
    1673             : static sql_rel *
    1674      230492 : rel_simplify_project_fk_join(mvc *sql, sql_rel *r, list *pexps, list *orderexps, int *changes)
    1675             : {
    1676      230492 :         sql_rel *rl = r->l, *rr = r->r, *nr = NULL;
    1677             :         sql_exp *je, *le, *nje, *re;
    1678             :         int fk_left = 1;
    1679             : 
    1680             :         /* check for foreign key join */
    1681      230492 :         if (list_length(r->exps) != 1)
    1682             :                 return r;
    1683      230492 :         if (!(je = exps_find_prop(r->exps, PROP_JOINIDX)) || je->flag != cmp_equal)
    1684             :                 return r;
    1685             :         /* je->l == foreign expression, je->r == primary expression */
    1686        1449 :         if (rel_find_exp(r->l, je->l)) {
    1687             :                 fk_left = 1;
    1688          41 :         } else if (rel_find_exp(r->r, je->l)) {
    1689             :                 fk_left = 0;
    1690             :         } else { /* not found */
    1691             :                 return r;
    1692             :         }
    1693             : 
    1694             :         /* primary side must be a full table */
    1695        1449 :         if ((fk_left && (!is_left(r->op) && !is_full(r->op)) && !is_basetable(rr->op)) ||
    1696          41 :                 (!fk_left && (!is_right(r->op) && !is_full(r->op)) && !is_basetable(rl->op)))
    1697         676 :                 return r;
    1698             : 
    1699         773 :         if (!check_projection_on_foreignside(r, pexps, fk_left) || !check_projection_on_foreignside(r, orderexps, fk_left))
    1700         731 :                 return r;
    1701             : 
    1702             :         /* rewrite, ie remove pkey side if possible */
    1703          42 :         le = (sql_exp*)je->l, re = (sql_exp*)je->l;
    1704             : 
    1705             :         /* both have NULL and there are semantics, the join cannot be removed */
    1706          42 :         if (is_semantics(je) && has_nil(le) && has_nil(re))
    1707             :                 return r;
    1708             : 
    1709          42 :         (*changes)++;
    1710             :         /* if the foreign key column doesn't have NULL values, then return it */
    1711          42 :         if (!has_nil(le) || is_full(r->op) || (fk_left && is_left(r->op)) || (!fk_left && is_right(r->op))) {
    1712          34 :                 if (fk_left) {
    1713          26 :                         nr = r->l;
    1714          26 :                         r->l = NULL;
    1715             :                 } else {
    1716           8 :                         nr = r->r;
    1717           8 :                         r->r = NULL;
    1718             :                 }
    1719          34 :                 rel_destroy(r);
    1720          34 :                 return nr;
    1721             :         }
    1722             : 
    1723             :         /* remove NULL values, ie generate a select not null */
    1724           8 :         nje = exp_compare(sql->sa, exp_ref(sql, le), exp_atom(sql->sa, atom_general(sql->sa, exp_subtype(le), NULL)), cmp_equal);
    1725           8 :         set_anti(nje);
    1726           8 :         set_has_no_nil(nje);
    1727           8 :         set_semantics(nje);
    1728           8 :         if (fk_left) {
    1729           8 :                 nr = r->l;
    1730           8 :                 r->l = NULL;
    1731             :         } else {
    1732           0 :                 nr = r->r;
    1733           0 :                 r->r = NULL;
    1734             :         }
    1735           8 :         rel_destroy(r);
    1736           8 :         return rel_select(sql->sa, nr, nje);
    1737             : }
    1738             : 
    1739             : static sql_rel *
    1740        1671 : rel_simplify_count_fk_join(mvc *sql, sql_rel *r, list *gexps, list *gcols, int *changes)
    1741             : {
    1742        1671 :         sql_rel *rl = r->l, *rr = r->r, *nr = NULL;
    1743             :         sql_exp *je, *le, *nje, *re, *oce;
    1744             :         int fk_left = 1;
    1745             : 
    1746             :         /* check for foreign key join */
    1747        1671 :         if (list_length(r->exps) != 1)
    1748             :                 return r;
    1749        1670 :         if (!(je = exps_find_prop(r->exps, PROP_JOINIDX)) || je->flag != cmp_equal)
    1750             :                 return r;
    1751             :         /* je->l == foreign expression, je->r == primary expression */
    1752          69 :         if (rel_find_exp(r->l, je->l)) {
    1753             :                 fk_left = 1;
    1754           7 :         } else if (rel_find_exp(r->r, je->l)) {
    1755             :                 fk_left = 0;
    1756             :         } else { /* not found */
    1757             :                 return r;
    1758             :         }
    1759             : 
    1760          69 :         oce = gexps->h->data;
    1761          69 :         if (oce->l) /* we only handle COUNT(*) */
    1762             :                 return r;
    1763             : 
    1764             :         /* primary side must be a full table */
    1765          68 :         if ((fk_left && (!is_left(r->op) && !is_full(r->op)) && !is_basetable(rr->op)) ||
    1766           6 :                 (!fk_left && (!is_right(r->op) && !is_full(r->op)) && !is_basetable(rl->op)))
    1767             :                 return r;
    1768             : 
    1769          39 :         if (fk_left && is_join(rl->op) && !rel_is_ref(rl)) {
    1770          13 :                 rl = rel_simplify_count_fk_join(sql, rl, gexps, gcols, changes);
    1771          13 :                 r->l = rl;
    1772             :         }
    1773          39 :         if (!fk_left && is_join(rr->op) && !rel_is_ref(rr)) {
    1774           2 :                 rr = rel_simplify_count_fk_join(sql, rr, gexps, gcols, changes);
    1775           2 :                 r->r = rr;
    1776             :         }
    1777             : 
    1778          39 :         if (!check_projection_on_foreignside(r, gcols, fk_left))
    1779             :                 return r;
    1780             : 
    1781             :         /* rewrite, ie remove pkey side if possible */
    1782          37 :         le = (sql_exp*)je->l, re = (sql_exp*)je->l;
    1783             : 
    1784             :         /* both have NULL and there are semantics, the join cannot be removed */
    1785          37 :         if (is_semantics(je) && has_nil(le) && has_nil(re))
    1786             :                 return r;
    1787             : 
    1788          37 :         (*changes)++;
    1789             :         /* if the foreign key column doesn't have NULL values, then return it */
    1790          37 :         if (!has_nil(le) || is_full(r->op) || (fk_left && is_left(r->op)) || (!fk_left && is_right(r->op))) {
    1791          31 :                 if (fk_left) {
    1792          25 :                         nr = r->l;
    1793          25 :                         r->l = NULL;
    1794             :                 } else {
    1795           6 :                         nr = r->r;
    1796           6 :                         r->r = NULL;
    1797             :                 }
    1798          31 :                 rel_destroy(r);
    1799          31 :                 return nr;
    1800             :         }
    1801             : 
    1802             :         /* remove NULL values, ie generate a select not null */
    1803           6 :         nje = exp_compare(sql->sa, exp_ref(sql, le), exp_atom(sql->sa, atom_general(sql->sa, exp_subtype(le), NULL)), cmp_equal);
    1804           6 :         set_anti(nje);
    1805           6 :         set_has_no_nil(nje);
    1806           6 :         set_semantics(nje);
    1807           6 :         if (fk_left) {
    1808           6 :                 nr = r->l;
    1809           6 :                 r->l = NULL;
    1810             :         } else {
    1811           0 :                 nr = r->r;
    1812           0 :                 r->r = NULL;
    1813             :         }
    1814           6 :         rel_destroy(r);
    1815           6 :         return rel_select(sql->sa, nr, nje);
    1816             : }
    1817             : 
    1818             : /*
    1819             :  * Handle (left/right/outer/natural) join fk-pk rewrites
    1820             :  *   1 group by ( fk-pk-join () ) [ count(*) ] -> group by ( fk )
    1821             :  *   2 project ( fk-pk-join () ) [ fk-column ] -> project (fk table)[ fk-column ]
    1822             :  *   3 project ( fk1-pk1-join( fk2-pk2-join()) [ fk-column, pk1 column ] -> project (fk1-pk1-join)[ fk-column, pk1 column ]
    1823             :  */
    1824             : static inline sql_rel *
    1825     4828566 : rel_simplify_fk_joins(visitor *v, sql_rel *rel)
    1826             : {
    1827             :         sql_rel *r = NULL;
    1828             : 
    1829     4828566 :         if (is_simple_project(rel->op))
    1830     1427335 :                 r = rel->l;
    1831             : 
    1832     4828608 :         while (is_simple_project(rel->op) && r && list_length(r->exps) == 1 && (is_join(r->op) || r->op == op_semi) && !(rel_is_ref(r))) {
    1833             :                 sql_rel *or = r;
    1834             : 
    1835      230492 :                 r = rel_simplify_project_fk_join(v->sql, r, rel->exps, rel->r, &v->changes);
    1836      230492 :                 if (r == or)
    1837             :                         return rel;
    1838          42 :                 rel->l = r;
    1839             :         }
    1840             : 
    1841     4598118 :         if (!is_groupby(rel->op))
    1842             :                 return rel;
    1843             : 
    1844      193136 :         r = rel->l;
    1845      294985 :         while(r && is_simple_project(r->op))
    1846      101849 :                 r = r->l;
    1847             : 
    1848      293080 :         while (is_groupby(rel->op) && !rel_is_ref(rel) && r && (is_join(r->op) || r->op == op_semi) && list_length(r->exps) == 1 && !(rel_is_ref(r)) &&
    1849             :                    /* currently only single count aggregation is handled, no other projects or aggregation */
    1850      130023 :                    list_length(rel->exps) == 1 && exp_aggr_is_count(rel->exps->h->data)) {
    1851             :                 sql_rel *or = r;
    1852             : 
    1853        1656 :                 r = rel_simplify_count_fk_join(v->sql, r, rel->exps, rel->r, &v->changes);
    1854        1656 :                 if (r == or)
    1855             :                         return rel;
    1856          25 :                 rel->l = r;
    1857             :         }
    1858             :         return rel;
    1859             : }
    1860             : 
    1861             : /*
    1862             :  * Push TopN (only LIMIT, no ORDER BY) down through projections underneath crossproduct, i.e.,
    1863             :  *
    1864             :  *     topn(                          topn(
    1865             :  *         project(                       project(
    1866             :  *             crossproduct(                  crossproduct(
    1867             :  *                 L,           =>                topn( L )[ n ],
    1868             :  *                 R                              topn( R )[ n ]
    1869             :  *             )                              )
    1870             :  *         )[ Cs ]*                       )[ Cs ]*
    1871             :  *     )[ n ]                         )[ n ]
    1872             :  *
    1873             :  *  (TODO: in case of n==1 we can omit the original top-level TopN)
    1874             :  *
    1875             :  * also push topn under (non reordering) projections.
    1876             :  */
    1877             : 
    1878             : static list *
    1879         317 : sum_limit_offset(mvc *sql, sql_rel *rel)
    1880             : {
    1881             :         /* for sample we always propagate */
    1882         317 :         if (is_sample(rel->op))
    1883           0 :                 return exps_copy(sql, rel->exps);
    1884             :         /* if the expression list only consists of a limit expression, we copy it */
    1885         317 :         if (list_length(rel->exps) == 1 && rel->exps->h->data)
    1886         311 :                 return list_append(sa_list(sql->sa), rel->exps->h->data);
    1887           6 :         sql_subtype *lng = sql_bind_localtype("lng");
    1888           6 :         sql_subfunc *add = sql_bind_func_result(sql, "sys", "sql_add", F_FUNC, lng, 2, lng, lng);
    1889           6 :         return list_append(sa_list(sql->sa), exp_op(sql->sa, rel->exps, add));
    1890             : }
    1891             : 
    1892             : static int
    1893        2577 : topn_sample_save_exps( list *exps )
    1894             : {
    1895             :         node *n;
    1896             : 
    1897             :         /* Limit only expression lists are always save */
    1898        2577 :         if (list_length(exps) == 1)
    1899             :                 return 1;
    1900         450 :         for (n = exps->h; n; n = n->next ) {
    1901         300 :                 sql_exp *e = n->data;
    1902             : 
    1903         300 :                 if (!e || e->type != e_atom)
    1904             :                         return 0;
    1905             :         }
    1906             :         return 1;
    1907             : }
    1908             : 
    1909             : static void
    1910          64 : rel_no_rename_exps( list *exps )
    1911             : {
    1912             :         node *n;
    1913             : 
    1914         371 :         for (n = exps->h; n; n = n->next) {
    1915         307 :                 sql_exp *e = n->data;
    1916             : 
    1917         307 :                 exp_setalias(e, e->l, e->r);
    1918             :         }
    1919          64 : }
    1920             : 
    1921             : static void
    1922       78230 : rel_rename_exps( mvc *sql, list *exps1, list *exps2)
    1923             : {
    1924             :         int pos = 0;
    1925             :         node *n, *m;
    1926             : 
    1927             :         (void)sql;
    1928             :         /* check if a column uses an alias earlier in the list */
    1929      667234 :         for (n = exps1->h, m = exps2->h; n && m; n = n->next, m = m->next, pos++) {
    1930      589004 :                 sql_exp *e2 = m->data;
    1931             : 
    1932      589004 :                 if (e2->type == e_column) {
    1933             :                         sql_exp *ne = NULL;
    1934             : 
    1935      545330 :                         if (e2->l)
    1936      511643 :                                 ne = exps_bind_column2(exps2, e2->l, e2->r, NULL);
    1937      545330 :                         if (!ne && !e2->l)
    1938       33687 :                                 ne = exps_bind_column(exps2, e2->r, NULL, NULL, 1);
    1939      545330 :                         if (ne) {
    1940      221719 :                                 int p = list_position(exps2, ne);
    1941             : 
    1942      221719 :                                 if (p < pos) {
    1943           8 :                                         ne = list_fetch(exps1, p);
    1944           8 :                                         if (e2->l)
    1945           8 :                                                 e2->l = (void *) exp_relname(ne);
    1946           8 :                                         e2->r = (void *) exp_name(ne);
    1947             :                                 }
    1948             :                         }
    1949             :                 }
    1950             :         }
    1951             : 
    1952       78230 :         assert(list_length(exps1) <= list_length(exps2));
    1953      667234 :         for (n = exps1->h, m = exps2->h; n && m; n = n->next, m = m->next) {
    1954      589004 :                 sql_exp *e1 = n->data;
    1955      589004 :                 sql_exp *e2 = m->data;
    1956      589004 :                 const char *rname = exp_relname(e1);
    1957             : 
    1958      589004 :                 if (!rname && e1->type == e_column && e1->l && exp_relname(e2) &&
    1959           0 :                     strcmp(e1->l, exp_relname(e2)) == 0)
    1960           0 :                         rname = exp_relname(e2);
    1961      589004 :                 exp_setalias(e2, rname, exp_name(e1));
    1962             :         }
    1963       78230 :         list_hash_clear(exps2);
    1964       78230 : }
    1965             : 
    1966             : static sql_rel *
    1967       38113 : rel_push_topn_and_sample_down(visitor *v, sql_rel *rel)
    1968             : {
    1969       38113 :         sql_rel *rp = NULL, *r = rel->l;
    1970             : 
    1971       38113 :         if ((is_topn(rel->op) || is_sample(rel->op)) && topn_sample_save_exps(rel->exps)) {
    1972        2577 :                 sql_rel *(*func) (sql_allocator *, sql_rel *, list *) = is_topn(rel->op) ? rel_topn : rel_sample;
    1973             : 
    1974             :                 /* nested topN relations */
    1975        2577 :                 if (r && is_topn(rel->op) && is_topn(r->op) && !rel_is_ref(r)) {
    1976           0 :                         sql_exp *topN1 = rel->exps->h->data, *topN2 = r->exps->h->data;
    1977           0 :                         sql_exp *offset1 = list_length(rel->exps) > 1 ? rel->exps->h->next->data : NULL;
    1978           0 :                         sql_exp *offset2 = list_length(r->exps) > 1 ? r->exps->h->next->data : NULL;
    1979             : 
    1980           0 :                         if (topN1->l && topN2->l && (!offset1 || offset1->l) && (!offset2 || offset2->l)) { /* no parameters */
    1981             :                                 bool changed = false;
    1982             : 
    1983           0 :                                 if ((!offset1 || (offset1->type == e_atom && offset1->l)) && (!offset2 || (offset2->type == e_atom && offset2->l))) { /* only atoms */
    1984           0 :                                         if (!offset1 && offset2) {
    1985           0 :                                                 list_append(rel->exps, exp_copy(v->sql, offset2));
    1986             :                                                 changed = true;
    1987           0 :                                         } else if (offset1 && offset2) { /* sum offsets */
    1988           0 :                                                 atom *b1 = (atom *)offset1->l, *b2 = (atom *)offset2->l, *c = atom_add(b1, b2);
    1989             : 
    1990           0 :                                                 if (!c) /* error, don't apply optimization, WARNING because of this the offset optimization must come before the limit one */
    1991             :                                                         return rel;
    1992           0 :                                                 if (atom_cmp(c, b2) < 0) /* overflow */
    1993           0 :                                                         c = atom_int(v->sql->sa, sql_bind_localtype("lng"), GDK_lng_max);
    1994           0 :                                                 offset1->l = c;
    1995             :                                                 changed = true;
    1996             :                                         }
    1997             :                                 }
    1998             : 
    1999           0 :                                 if (topN1->type == e_atom && topN1->l && topN2->type == e_atom && topN2->l) { /* only atoms */
    2000             :                                         atom *a1 = (atom *)topN1->l, *a2 = (atom *)topN2->l;
    2001             : 
    2002           0 :                                         if (!a2->isnull && (a1->isnull || atom_cmp(a1, a2) >= 0)) { /* topN1 is not set or is larger than topN2 */
    2003           0 :                                                 rel->exps->h->data = exp_copy(v->sql, topN2);
    2004             :                                                 changed = true;
    2005             :                                         }
    2006             :                                 }
    2007             : 
    2008           0 :                                 if (changed) {
    2009           0 :                                         rel->l = r->l;
    2010           0 :                                         r->l = NULL;
    2011           0 :                                         rel_destroy(r);
    2012           0 :                                         v->changes++;
    2013           0 :                                         return rel;
    2014             :                                 }
    2015             :                         }
    2016             :                 }
    2017             : 
    2018        2577 :                 if (r && is_simple_project(r->op) && need_distinct(r))
    2019             :                         return rel;
    2020             : 
    2021             :                 /* push topn/sample under projections */
    2022        2573 :                 if (!rel_is_ref(rel) && r && is_simple_project(r->op) && !need_distinct(r) && !rel_is_ref(r) && r->l && !r->r) {
    2023             :                         sql_rel *x = r, *px = x;
    2024             : 
    2025         688 :                         while (is_simple_project(x->op) && !need_distinct(x) && !rel_is_ref(x) && x->l && !x->r) {
    2026             :                                 px = x;
    2027             :                                 x = x->l;
    2028             :                         }
    2029             :                         /* only push topn once */
    2030         328 :                         if (x && x->op == rel->op)
    2031             :                                 return rel;
    2032             : 
    2033         315 :                         rel->l = x;
    2034         315 :                         px->l = rel;
    2035             :                         rel = r;
    2036         315 :                         v->changes++;
    2037         315 :                         return rel;
    2038             :                 }
    2039             : 
    2040             :                 /* duplicate topn/sample direct under union or crossproduct */
    2041        2245 :                 if (r && !rel_is_ref(r) && r->l && r->r && ((is_union(r->op) && r->exps) || (r->op == op_join && list_empty(r->exps)))) {
    2042             :                         sql_rel *u = r, *x;
    2043         260 :                         sql_rel *ul = u->l;
    2044         260 :                         sql_rel *ur = u->r;
    2045             :                         bool changed = false;
    2046             : 
    2047             :                         x = ul;
    2048         365 :                         while (is_simple_project(x->op) && !need_distinct(x) && !rel_is_ref(x) && x->l && !x->r)
    2049             :                                 x = x->l;
    2050         260 :                         if (x && x->op != rel->op) { /* only push topn once */
    2051          95 :                                 ul = func(v->sql->sa, ul, sum_limit_offset(v->sql, rel));
    2052          95 :                                 u->l = ul;
    2053             :                                 changed = true;
    2054             :                         }
    2055             : 
    2056             :                         x = ur;
    2057         340 :                         while (is_simple_project(x->op) && !need_distinct(x) && !rel_is_ref(x) && x->l && !x->r)
    2058             :                                 x = x->l;
    2059         260 :                         if (x && x->op != rel->op) { /* only push topn once */
    2060          94 :                                 ur = func(v->sql->sa, ur, sum_limit_offset(v->sql, rel));
    2061          94 :                                 u->r = ur;
    2062             :                                 changed = true;
    2063             :                         }
    2064             : 
    2065         166 :                         if (changed)
    2066          96 :                                 v->changes++;
    2067         260 :                         return rel;
    2068             :                 }
    2069             : 
    2070             :                 /* duplicate topn/sample + [ project-order ] under union */
    2071        1985 :                 if (r)
    2072        1985 :                         rp = r->l;
    2073        1985 :                 if (r && r->exps && is_simple_project(r->op) && !rel_is_ref(r) && r->r && r->l && is_union(rp->op)) {
    2074             :                         sql_rel *u = rp, *ou = u, *x;
    2075         155 :                         sql_rel *ul = u->l;
    2076         155 :                         sql_rel *ur = u->r;
    2077             :                         int add_r = 0;
    2078             :                         list *rcopy = NULL;
    2079             : 
    2080             :                         /* only push topn/sample once */
    2081             :                         x = ul;
    2082         188 :                         while (is_simple_project(x->op) && !need_distinct(x) && !rel_is_ref(x) && x->l && !x->r)
    2083             :                                 x = x->l;
    2084         155 :                         if (x && x->op == rel->op)
    2085             :                                 return rel;
    2086             :                         x = ur;
    2087         141 :                         while (is_simple_project(x->op) && !need_distinct(x) && !rel_is_ref(x) && x->l && !x->r)
    2088             :                                 x = x->l;
    2089          64 :                         if (x && x->op == rel->op)
    2090             :                                 return rel;
    2091             : 
    2092          64 :                         if (list_length(ul->exps) > list_length(r->exps)) {
    2093             :                                 add_r = 1;
    2094           1 :                                 rcopy = exps_copy(v->sql, r->r);
    2095           2 :                                 for (node *n = rcopy->h ; n ; n = n->next) {
    2096           1 :                                         sql_exp *e = n->data;
    2097           1 :                                         set_descending(e);
    2098           1 :                                         set_nulls_first(e);
    2099             :                                 }
    2100             :                         }
    2101          64 :                         ul = rel_dup(ul);
    2102          64 :                         ur = rel_dup(ur);
    2103          64 :                         if (!is_project(ul->op))
    2104           0 :                                 ul = rel_project(v->sql->sa, ul,
    2105             :                                         rel_projections(v->sql, ul, NULL, 1, 1));
    2106          64 :                         if (!is_project(ur->op))
    2107           0 :                                 ur = rel_project(v->sql->sa, ur,
    2108             :                                         rel_projections(v->sql, ur, NULL, 1, 1));
    2109          64 :                         rel_rename_exps(v->sql, u->exps, ul->exps);
    2110          64 :                         rel_rename_exps(v->sql, u->exps, ur->exps);
    2111             : 
    2112             :                         /* introduce projects under the set */
    2113          64 :                         ul = rel_project(v->sql->sa, ul, NULL);
    2114          64 :                         ul->exps = exps_copy(v->sql, r->exps);
    2115             :                         /* possibly add order by column */
    2116          64 :                         if (add_r)
    2117           1 :                                 ul->exps = list_distinct(list_merge(ul->exps, exps_copy(v->sql, rcopy), NULL), (fcmp) exp_equal, (fdup) NULL);
    2118          64 :                         ul->nrcols = list_length(ul->exps);
    2119          64 :                         ul->r = exps_copy(v->sql, r->r);
    2120          64 :                         ul = func(v->sql->sa, ul, sum_limit_offset(v->sql, rel));
    2121             : 
    2122          64 :                         ur = rel_project(v->sql->sa, ur, NULL);
    2123          64 :                         ur->exps = exps_copy(v->sql, r->exps);
    2124             :                         /* possibly add order by column */
    2125          64 :                         if (add_r)
    2126           1 :                                 ur->exps = list_distinct(list_merge(ur->exps, exps_copy(v->sql, rcopy), NULL), (fcmp) exp_equal, (fdup) NULL);
    2127          64 :                         ur->nrcols = list_length(ur->exps);
    2128          64 :                         ur->r = exps_copy(v->sql, r->r);
    2129          64 :                         ur = func(v->sql->sa, ur, sum_limit_offset(v->sql, rel));
    2130             : 
    2131          64 :                         u = rel_setop(v->sql->sa, ul, ur, op_union);
    2132          64 :                         u->exps = exps_alias(v->sql, r->exps);
    2133          64 :                         u->nrcols = list_length(u->exps);
    2134          64 :                         set_processed(u);
    2135             :                         /* possibly add order by column */
    2136          64 :                         if (add_r)
    2137           1 :                                 u->exps = list_distinct(list_merge(u->exps, rcopy, NULL), (fcmp) exp_equal, (fdup) NULL);
    2138          64 :                         if (need_distinct(r)) {
    2139           0 :                                 set_distinct(ul);
    2140           0 :                                 set_distinct(ur);
    2141             :                         }
    2142             : 
    2143             :                         /* zap names */
    2144          64 :                         rel_no_rename_exps(u->exps);
    2145          64 :                         rel_destroy(ou);
    2146             : 
    2147          64 :                         ur = rel_project(v->sql->sa, u, exps_alias(v->sql, r->exps));
    2148          64 :                         ur->r = r->r;
    2149          64 :                         r->l = NULL;
    2150             : 
    2151          64 :                         if (need_distinct(r))
    2152           0 :                                 set_distinct(ur);
    2153             : 
    2154          64 :                         rel_destroy(r);
    2155          64 :                         rel->l = ur;
    2156          64 :                         v->changes++;
    2157          64 :                         return rel;
    2158             :                 }
    2159             :         }
    2160             :         return rel;
    2161             : }
    2162             : 
    2163             : /* merge projection */
    2164             : 
    2165             : /* push an expression through a projection.
    2166             :  * The result should again used in a projection.
    2167             :  */
    2168             : static sql_exp *
    2169             : exp_push_down_prj(mvc *sql, sql_exp *e, sql_rel *f, sql_rel *t);
    2170             : 
    2171             : static list *
    2172       80497 : exps_push_down_prj(mvc *sql, list *exps, sql_rel *f, sql_rel *t)
    2173             : {
    2174             :         node *n;
    2175       80497 :         list *nl = new_exp_list(sql->sa);
    2176             : 
    2177      213195 :         for(n = exps->h; n; n = n->next) {
    2178      151231 :                 sql_exp *arg = n->data, *narg = NULL;
    2179             : 
    2180      151231 :                 narg = exp_push_down_prj(sql, arg, f, t);
    2181      151231 :                 if (!narg)
    2182             :                         return NULL;
    2183      132698 :                 narg = exp_propagate(sql->sa, narg, arg);
    2184      132698 :                 append(nl, narg);
    2185             :         }
    2186             :         return nl;
    2187             : }
    2188             : 
    2189             : static sql_exp *
    2190     2494332 : exp_push_down_prj(mvc *sql, sql_exp *e, sql_rel *f, sql_rel *t)
    2191             : {
    2192             :         sql_exp *ne = NULL, *l, *r, *r2;
    2193             : 
    2194     2494332 :         assert(is_project(f->op));
    2195             : 
    2196     2494332 :         switch(e->type) {
    2197     2053312 :         case e_column:
    2198     2053312 :                 if (e->l)
    2199     1991892 :                         ne = exps_bind_column2(f->exps, e->l, e->r, NULL);
    2200     2053312 :                 if (!ne && !e->l)
    2201       61420 :                         ne = exps_bind_column(f->exps, e->r, NULL, NULL, 1);
    2202     2053312 :                 if (!ne || (ne->type != e_column && (ne->type != e_atom || ne->f)))
    2203             :                         return NULL;
    2204     1796864 :                 while (ne && has_label(ne) && f->op == op_project && ne->type == e_column) {
    2205             :                         sql_exp *oe = e, *one = ne;
    2206             : 
    2207             :                         e = ne;
    2208             :                         ne = NULL;
    2209       12438 :                         if (e->l)
    2210       12438 :                                 ne = exps_bind_column2(f->exps, e->l, e->r, NULL);
    2211       12438 :                         if (!ne && !e->l)
    2212           0 :                                 ne = exps_bind_column(f->exps, e->r, NULL, NULL, 1);
    2213       12438 :                         if (ne && ne != one && list_position(f->exps, ne) >= list_position(f->exps, one))
    2214             :                                 ne = NULL;
    2215       12438 :                         if (!ne || ne == one) {
    2216             :                                 ne = one;
    2217             :                                 e = oe;
    2218             :                                 break;
    2219             :                         }
    2220         480 :                         if (ne->type != e_column && (ne->type != e_atom || ne->f))
    2221             :                                 return NULL;
    2222             :                 }
    2223             :                 /* possibly a groupby/project column is renamed */
    2224     1796384 :                 if (is_groupby(f->op) && f->r) {
    2225             :                         sql_exp *gbe = NULL;
    2226          12 :                         if (ne->l)
    2227          12 :                                 gbe = exps_bind_column2(f->r, ne->l, ne->r, NULL);
    2228          12 :                         if (!gbe && !e->l)
    2229           0 :                                 gbe = exps_bind_column(f->r, ne->r, NULL, NULL, 1);
    2230             :                         ne = gbe;
    2231          12 :                         if (!ne || (ne->type != e_column && (ne->type != e_atom || ne->f)))
    2232             :                                 return NULL;
    2233             :                 }
    2234     1796384 :                 if (ne->type == e_atom)
    2235       34670 :                         e = exp_copy(sql, ne);
    2236             :                 else
    2237     1761714 :                         e = exp_alias(sql->sa, exp_relname(e), exp_name(e), ne->l, ne->r, exp_subtype(e), e->card, has_nil(e), is_unique(e), is_intern(e));
    2238     1796384 :                 return exp_propagate(sql->sa, e, ne);
    2239       73024 :         case e_cmp:
    2240       73024 :                 if (e->flag == cmp_or || e->flag == cmp_filter) {
    2241        5739 :                         list *l = exps_push_down_prj(sql, e->l, f, t);
    2242        5739 :                         list *r = exps_push_down_prj(sql, e->r, f, t);
    2243             : 
    2244        5739 :                         if (!l || !r)
    2245             :                                 return NULL;
    2246        1900 :                         if (e->flag == cmp_filter)
    2247         115 :                                 return exp_filter(sql->sa, l, r, e->f, is_anti(e));
    2248        1785 :                         return exp_or(sql->sa, l, r, is_anti(e));
    2249       67285 :                 } else if (e->flag == cmp_in || e->flag == cmp_notin) {
    2250        4596 :                         sql_exp *l = exp_push_down_prj(sql, e->l, f, t);
    2251        4596 :                         list *r = exps_push_down_prj(sql, e->r, f, t);
    2252             : 
    2253        4596 :                         if (!l || !r)
    2254             :                                 return NULL;
    2255         435 :                         return exp_in(sql->sa, l, r, e->flag);
    2256             :                 } else {
    2257       62689 :                         l = exp_push_down_prj(sql, e->l, f, t);
    2258       62689 :                         r = exp_push_down_prj(sql, e->r, f, t);
    2259       62689 :                         if (e->f) {
    2260        6072 :                                 r2 = exp_push_down_prj(sql, e->f, f, t);
    2261        6072 :                                 if (l && r && r2)
    2262         491 :                                         ne = exp_compare2(sql->sa, l, r, r2, e->flag, is_symmetric(e));
    2263       56617 :                         } else if (l && r) {
    2264       30238 :                                 ne = exp_compare(sql->sa, l, r, e->flag);
    2265             :                         }
    2266             :                 }
    2267       30729 :                 if (!ne)
    2268       31960 :                         return NULL;
    2269       30729 :                 return exp_propagate(sql->sa, ne, e);
    2270      198740 :         case e_convert:
    2271      198740 :                 l = exp_push_down_prj(sql, e->l, f, t);
    2272      198740 :                 if (l)
    2273      136855 :                         return exp_convert(sql->sa, l, exp_fromtype(e), exp_totype(e));
    2274             :                 return NULL;
    2275       64423 :         case e_aggr:
    2276             :         case e_func: {
    2277       64423 :                 list *l = e->l, *nl = NULL;
    2278             :                 sql_exp *ne = NULL;
    2279             : 
    2280       64423 :                 if (e->type == e_func && exp_unsafe(e,0))
    2281             :                         return NULL;
    2282       64423 :                 if (!list_empty(l)) {
    2283       64423 :                         nl = exps_push_down_prj(sql, l, f, t);
    2284       64423 :                         if (!nl)
    2285             :                                 return NULL;
    2286             :                 }
    2287       53105 :                 if (e->type == e_func)
    2288       53105 :                         ne = exp_op(sql->sa, nl, e->f);
    2289             :                 else
    2290           0 :                         ne = exp_aggr(sql->sa, nl, e->f, need_distinct(e), need_no_nil(e), e->card, has_nil(e));
    2291       53105 :                 return exp_propagate(sql->sa, ne, e);
    2292             :         }
    2293      104833 :         case e_atom: {
    2294      104833 :                 list *l = e->f, *nl = NULL;
    2295             : 
    2296      104833 :                 if (!list_empty(l)) {
    2297           0 :                         nl = exps_push_down_prj(sql, l, f, t);
    2298           0 :                         if (!nl)
    2299             :                                 return NULL;
    2300           0 :                         ne = exp_values(sql->sa, nl);
    2301             :                 } else {
    2302      104833 :                         ne = exp_copy(sql, e);
    2303             :                 }
    2304      104833 :                 return exp_propagate(sql->sa, ne, e);
    2305             :         }
    2306           0 :         case e_psm:
    2307           0 :                 if (e->type == e_atom && e->f) /* value list */
    2308           0 :                         return NULL;
    2309             :                 return e;
    2310             :         }
    2311             :         return NULL;
    2312             : }
    2313             : 
    2314             : static int
    2315         208 : rel_is_unique(sql_rel *rel)
    2316             : {
    2317         217 :         switch(rel->op) {
    2318           9 :         case op_semi:
    2319             :         case op_anti:
    2320             :         case op_inter:
    2321             :         case op_except:
    2322             :         case op_topn:
    2323             :         case op_sample:
    2324           9 :                 return rel_is_unique(rel->l);
    2325             :         case op_table:
    2326             :         case op_basetable:
    2327             :                 return 1;
    2328         206 :         default:
    2329         206 :                 return 0;
    2330             :         }
    2331             : }
    2332             : 
    2333             : /* WARNING exps_unique doesn't check for duplicate NULL values */
    2334             : int
    2335       70555 : exps_unique(mvc *sql, sql_rel *rel, list *exps)
    2336             : {
    2337             :         int nr = 0, need_check = 0;
    2338             :         sql_ukey *k = NULL;
    2339             : 
    2340       70555 :         if (list_empty(exps))
    2341             :                 return 0;
    2342      182617 :         for(node *n = exps->h; n ; n = n->next) {
    2343      112062 :                 sql_exp *e = n->data;
    2344             :                 prop *p;
    2345             : 
    2346      112062 :                 if (!is_unique(e)) { /* ignore unique columns */
    2347      111460 :                         need_check++;
    2348      111460 :                         if (!k && (p = find_prop(e->p, PROP_HASHCOL))) /* at the moment, use only one k */
    2349         262 :                                 k = p->value;
    2350             :                 }
    2351             :         }
    2352       70555 :         if (!need_check) /* all have unique property return */
    2353             :                 return 1;
    2354       70419 :         if (!k || list_length(k->k.columns) != need_check)
    2355       70202 :                 return 0;
    2356         217 :         if (rel) {
    2357         217 :                 char *matched = SA_ZNEW_ARRAY(sql->sa, char, list_length(k->k.columns));
    2358             :                 fcmp cmp = (fcmp)&kc_column_cmp;
    2359         442 :                 for(node *n = exps->h; n; n = n->next) {
    2360         225 :                         sql_exp *e = n->data;
    2361             :                         sql_column *c;
    2362             :                         node *m;
    2363             : 
    2364         225 :                         if (is_unique(e))
    2365           4 :                                 continue;
    2366         221 :                         if ((c = exp_find_column(rel, e, -2)) != NULL && (m = list_find(k->k.columns, c, cmp)) != NULL) {
    2367         212 :                                 int pos = list_position(k->k.columns, m->data);
    2368         212 :                                 if (!matched[pos])
    2369         212 :                                         nr++;
    2370         212 :                                 matched[pos] = 1;
    2371             :                         }
    2372             :                 }
    2373         217 :                 if (nr == list_length(k->k.columns))
    2374         208 :                         return rel_is_unique(rel);
    2375             :         }
    2376             :         return 0;
    2377             : }
    2378             : 
    2379             : static sql_column *
    2380      817485 : exp_is_pkey(sql_rel *rel, sql_exp *e)
    2381             : {
    2382      817485 :         if (find_prop(e->p, PROP_HASHCOL)) { /* aligned PKEY JOIN */
    2383             :                 fcmp cmp = (fcmp)&kc_column_cmp;
    2384       18596 :                 sql_column *c = exp_find_column(rel, e, -2);
    2385             : 
    2386       18596 :                 if (c && c->t->pkey && list_find(c->t->pkey->k.columns, c, cmp) != NULL)
    2387       18596 :                         return c;
    2388             :         }
    2389             :         return NULL;
    2390             : }
    2391             : 
    2392             : static sql_exp *
    2393      359110 : rel_is_join_on_pkey(sql_rel *rel, bool pk_fk) /* pk_fk is used to verify is a join on pk-fk */
    2394             : {
    2395      359110 :         if (!rel || !rel->exps)
    2396             :                 return NULL;
    2397      780773 :         for (node *n = rel->exps->h; n; n = n->next) {
    2398      422767 :                 sql_exp *je = n->data;
    2399             : 
    2400      831915 :                 if (je->type == e_cmp && je->flag == cmp_equal &&
    2401      818296 :                         (exp_is_pkey(rel, je->l) || exp_is_pkey(rel, je->r)) &&
    2402       18595 :                         (!pk_fk || find_prop(je->p, PROP_JOINIDX)))
    2403           0 :                         return je;
    2404             :         }
    2405             :         return NULL;
    2406             : }
    2407             : 
    2408             : /* if all arguments to a distinct aggregate are unique, remove 'distinct' property */
    2409             : static inline sql_rel *
    2410      200583 : rel_distinct_aggregate_on_unique_values(visitor *v, sql_rel *rel)
    2411             : {
    2412      200583 :         if (is_groupby(rel->op) && !list_empty(rel->exps)) {
    2413      605102 :                 for (node *n = rel->exps->h; n; n = n->next) {
    2414      404519 :                         sql_exp *exp = (sql_exp*) n->data;
    2415             : 
    2416      404519 :                         if (exp->type == e_aggr && need_distinct(exp)) {
    2417             :                                 bool all_unique = true;
    2418        3866 :                                 list *l = exp->l;
    2419             : 
    2420        7732 :                                 for (node *m = l->h; m && all_unique; m = m->next) {
    2421        3866 :                                         sql_exp *arg = (sql_exp*) m->data;
    2422             : 
    2423        7627 :                                         all_unique &= arg->type == e_column && is_unique(arg) && (!is_semantics(exp) || !has_nil(arg));
    2424             :                                 }
    2425        3866 :                                 if (!all_unique && exps_card(l) > CARD_ATOM)
    2426        3657 :                                         all_unique = exps_unique(v->sql, rel, l) && (!is_semantics(exp) || !have_nil(l));
    2427        3866 :                                 if (all_unique) {
    2428         105 :                                         set_nodistinct(exp);
    2429         105 :                                         v->changes++;
    2430             :                                 }
    2431             :                         }
    2432             :                 }
    2433             :         }
    2434      200583 :         return rel;
    2435             : }
    2436             : 
    2437             : static bool
    2438           0 : has_no_selectivity(mvc *sql, sql_rel *rel)
    2439             : {
    2440           0 :         if (!rel)
    2441             :                 return true;
    2442             : 
    2443           0 :         switch(rel->op){
    2444             :         case op_basetable:
    2445             :         case op_truncate:
    2446             :         case op_table:
    2447             :                 return true;
    2448           0 :         case op_topn:
    2449             :         case op_sample:
    2450             :         case op_project:
    2451             :         case op_groupby:
    2452           0 :                 return has_no_selectivity(sql, rel->l);
    2453           0 :         case op_ddl:
    2454             :         case op_insert:
    2455             :         case op_update:
    2456             :         case op_delete:
    2457             :         case op_merge:
    2458             :         case op_join:
    2459             :         case op_left:
    2460             :         case op_right:
    2461             :         case op_full:
    2462             :         case op_semi:
    2463             :         case op_anti:
    2464             :         case op_union:
    2465             :         case op_inter:
    2466             :         case op_except:
    2467             :         case op_select:
    2468           0 :                 return false;
    2469             :         }
    2470             :         return rel;
    2471             : }
    2472             : 
    2473             : /*
    2474             :  * Remove a redundant join
    2475             :  *
    2476             :  * join (L, Distinct Project(join(L,P) [ p.key == l.lkey]) [p.key]) [ p.key == l.lkey]
    2477             :  * =>
    2478             :  * join(L, P) [p.key==l.lkey]
    2479             :  */
    2480             : static sql_rel *
    2481     2026757 : rel_remove_redundant_join(visitor *v, sql_rel *rel)
    2482             : {
    2483     2026757 :         if ((is_join(rel->op) || is_semi(rel->op)) && !list_empty(rel->exps)) {
    2484      362554 :                 sql_rel *l = rel->l, *r = rel->r, *b, *p = NULL, *j;
    2485             : 
    2486      362554 :                 if (is_basetable(l->op) && is_simple_project(r->op) && need_distinct(r)) {
    2487             :                         b = l;
    2488             :                         p = r;
    2489          14 :                         j = p->l;
    2490      362540 :                 } else if (is_basetable(r->op) && is_simple_project(l->op) && need_distinct(l)) {
    2491             :                         b = r;
    2492             :                         p = l;
    2493        9134 :                         j = p->l;
    2494             :                 }
    2495      362554 :                 if (!p || !j || j->op != rel->op)
    2496             :                         return rel;
    2497             :                 /* j must have b->l (ie table) */
    2498          32 :                 sql_rel *jl = j->l, *jr = j->r;
    2499          32 :                 if ((is_basetable(jl->op) && jl->l == b->l) ||
    2500          25 :                     (is_basetable(jr->op) && jr->l == b->l)) {
    2501             :                         int left = 0;
    2502          14 :                         if (is_basetable(jl->op) && jl->l == b->l)
    2503             :                                 left = 1;
    2504          14 :                         if (!list_empty(p->exps)) {
    2505          22 :                                 for (node *n=p->exps->h; n; n = n->next) { /* all exps of 'p' must be bound to the opposite side */
    2506          14 :                                         sql_exp *e = n->data;
    2507             : 
    2508          21 :                                         if (!rel_rebind_exp(v->sql, left ? jr : jl, e))
    2509             :                                                 return rel;
    2510             :                                 }
    2511             :                         }
    2512           8 :                         if (exp_match_list(j->exps, rel->exps)) {
    2513           1 :                                 p->l = (left)?rel_dup(jr):rel_dup(jl);
    2514           1 :                                 rel_destroy(j);
    2515           1 :                                 set_nodistinct(p);
    2516           1 :                                 v->changes++;
    2517           1 :                                 return rel;
    2518             :                         }
    2519             :                 }
    2520             :         }
    2521             :         return rel;
    2522             : }
    2523             : 
    2524             : static sql_column *
    2525           0 : is_fk_column_of_pk(mvc *sql, sql_rel *rel, sql_column *pkc, sql_exp *e) /* test if e is a foreing key column for the pk on pkc */
    2526             : {
    2527           0 :         sql_trans *tr = sql->session->tr;
    2528           0 :         sql_column *c = exp_find_column(rel, e, -2);
    2529             : 
    2530           0 :         if (c) {
    2531           0 :                 sql_table *t = c->t;
    2532             : 
    2533           0 :                 for (node *n = ol_first_node(t->idxs); n; n = n->next) {
    2534           0 :                         sql_idx *li = n->data;
    2535             : 
    2536           0 :                         if (li->type == join_idx) {
    2537           0 :                                 for (node *m = li->columns->h ; m ; m = m->next) {
    2538           0 :                                         sql_kc *fkc = m->data;
    2539             : 
    2540           0 :                                         if (strcmp(fkc->c->base.name, c->base.name) == 0) { /* same fkey column */
    2541           0 :                                                 sql_key *fkey = (sql_key*)os_find_id(tr->cat->objects, tr, ((sql_fkey*)li->key)->rkey);
    2542             : 
    2543           0 :                                                 if (strcmp(fkey->t->base.name, pkc->t->base.name) == 0) { /* to same pk table */
    2544           0 :                                                         for (node *o = fkey->columns->h ; o ; o = n->next) {
    2545           0 :                                                                 sql_kc *kc = m->data;
    2546             : 
    2547           0 :                                                                 if (strcmp(kc->c->base.name, pkc->base.name) == 0) /* to same pk table column */
    2548             :                                                                         return c;
    2549             :                                                         }
    2550             :                                                 }
    2551             :                                         }
    2552             :                                 }
    2553             :                         }
    2554             :                 }
    2555             :         }
    2556             :         return NULL;
    2557             : }
    2558             : 
    2559             : static sql_rel *
    2560     1232830 : rel_distinct_project2groupby(visitor *v, sql_rel *rel)
    2561             : {
    2562     1232830 :         sql_rel *l = rel->l;
    2563             : 
    2564             :         /* rewrite distinct project (table) [ constant ] -> project [ constant ] */
    2565     1240058 :         if (rel->op == op_project && rel->l && !rel->r /* no order by */ && need_distinct(rel) &&
    2566        7228 :             exps_card(rel->exps) <= CARD_ATOM) {
    2567         183 :                 set_nodistinct(rel);
    2568         183 :                 if (rel->card > CARD_ATOM) /* if the projection just contains constants, then no topN is needed */
    2569          66 :                         rel->l = rel_topn(v->sql->sa, rel->l, append(sa_list(v->sql->sa), exp_atom_lng(v->sql->sa, 1)));
    2570         183 :                 v->changes++;
    2571             :         }
    2572             : 
    2573             :         /* rewrite distinct project [ pk ] ( select ( table ) [ e op val ])
    2574             :          * into project [ pk ] ( select/semijoin ( table )  */
    2575     1232830 :         if (rel->op == op_project && rel->l && !rel->r /* no order by */ && need_distinct(rel) &&
    2576        7046 :             (l->op == op_select || l->op == op_semi) && exps_unique(v->sql, rel, rel->exps) &&
    2577           1 :                 (!have_semantics(l->exps) || !have_nil(rel->exps))) {
    2578           1 :                 set_nodistinct(rel);
    2579           1 :                 v->changes++;
    2580             :         }
    2581             : 
    2582             :         /* rewrite distinct project ( join(p,f) [ p.pk = f.fk ] ) [ p.pk ]
    2583             :          *      into project( (semi)join(p,f) [ p.pk = f.fk ] ) [ p.pk ] */
    2584     1232830 :         if (rel->op == op_project && rel->l && !rel->r /* no order by */ && need_distinct(rel) &&
    2585        7044 :             l && (is_select(l->op) || l->op == op_join) && rel_is_join_on_pkey(l, true) /* [ pk == fk ] */) {
    2586             :                 sql_exp *found = NULL, *pk = NULL, *fk = NULL;
    2587             :                 bool all_exps_atoms = true;
    2588             :                 sql_column *pkc = NULL;
    2589             : 
    2590           0 :                 for (node *m = l->exps->h ; m ; m = m->next) { /* find a primary key join */
    2591           0 :                         sql_exp *je = (sql_exp *) m->data;
    2592           0 :                         sql_exp *le = je->l, *re = je->r;
    2593             : 
    2594           0 :                         if (!find_prop(je->p, PROP_JOINIDX)) /* must be a pk-fk join expression */
    2595           0 :                                 continue;
    2596             : 
    2597           0 :                         if ((pkc = exp_is_pkey(l, le))) { /* le is the primary key */
    2598             :                                 all_exps_atoms = true;
    2599             : 
    2600           0 :                                 for (node *n = rel->exps->h; n && all_exps_atoms; n = n->next) {
    2601           0 :                                         sql_exp *e = (sql_exp *) n->data;
    2602             : 
    2603           0 :                                         if (exp_match(e, le) || exp_refers(e, le))
    2604             :                                                 found = e;
    2605           0 :                                         else if (e->card > CARD_ATOM)
    2606             :                                                 all_exps_atoms = false;
    2607             :                                 }
    2608             :                                 pk = le;
    2609             :                                 fk = re;
    2610             :                         }
    2611           0 :                         if (!found && (pkc = exp_is_pkey(l, re))) { /* re is the primary key */
    2612             :                                 all_exps_atoms = true;
    2613             : 
    2614           0 :                                 for (node *n = rel->exps->h; n && all_exps_atoms; n = n->next) {
    2615           0 :                                         sql_exp *e = (sql_exp *) n->data;
    2616             : 
    2617           0 :                                         if (exp_match(e, re) || exp_refers(e, re))
    2618             :                                                 found = e;
    2619           0 :                                         else if (e->card > CARD_ATOM)
    2620             :                                                 all_exps_atoms = false;
    2621             :                                 }
    2622             :                                 pk = re;
    2623             :                                 fk = le;
    2624             :                         }
    2625             :                 }
    2626             : 
    2627           0 :                 if (all_exps_atoms && found) { /* rel must have the same primary key on the projection list */
    2628             :                         /* if the foreign key has no selectivity, the join can be removed */
    2629           0 :                         if (!(rel_is_ref(l)) && ((rel_find_exp(l->l, fk) && is_fk_column_of_pk(v->sql, l->l, pkc, fk) && has_no_selectivity(v->sql, l->l)) ||
    2630           0 :                                 (l->r && rel_find_exp(l->r, fk) && is_fk_column_of_pk(v->sql, l->r, pkc, fk) && has_no_selectivity(v->sql, l->r)))) {
    2631           0 :                                 sql_rel *side = (rel_find_exp(l->l, pk) != NULL)?l->l:l->r;
    2632             : 
    2633           0 :                                 rel->l = rel_dup(side);
    2634           0 :                                 rel_destroy(l);
    2635           0 :                                 v->changes++;
    2636           0 :                                 set_nodistinct(rel);
    2637           0 :                                 return rel;
    2638             :                         }
    2639             :                         /* if the join has no multiple references it can be re-written into a semijoin */
    2640           0 :                         if (l->op == op_join && !(rel_is_ref(l)) && list_length(rel->exps) == 1) { /* other expressions may come from the other side */
    2641           0 :                                 if (l->r && rel_find_exp(l->r, pk)) {
    2642           0 :                                         sql_rel *temp = l->l;
    2643           0 :                                         l->l = l->r;
    2644           0 :                                         l->r = temp;
    2645             : 
    2646           0 :                                         l->op = op_semi;
    2647           0 :                                 } else if (rel_find_exp(l->l, pk)) {
    2648           0 :                                         l->op = op_semi;
    2649             :                                 }
    2650             :                         }
    2651           0 :                         v->changes++;
    2652           0 :                         set_nodistinct(rel);
    2653           0 :                         return rel;
    2654             :                 }
    2655             :         }
    2656             :         /* rewrite distinct project [ gbe ] ( select ( groupby [ gbe ] [ gbe, e ] )[ e op val ])
    2657             :          * into project [ gbe ] ( select ( group etc ) */
    2658     1232830 :         if (rel->op == op_project && rel->l && !rel->r /* no order by */ &&
    2659        7044 :             need_distinct(rel) && l->op == op_select){
    2660        2575 :                 sql_rel *g = l->l;
    2661        2575 :                 if (is_groupby(g->op)) {
    2662          12 :                         list *used = sa_list(v->sql->sa);
    2663          12 :                         list *gbe = g->r;
    2664             :                         node *n;
    2665             :                         int fnd = 1;
    2666             : 
    2667          24 :                         for (n = rel->exps->h; n && fnd; n = n->next) {
    2668          12 :                                 sql_exp *e = n->data;
    2669             : 
    2670          12 :                                 if (e->card > CARD_ATOM) {
    2671             :                                         /* find e in gbe */
    2672          12 :                                         sql_exp *ne = list_find_exp(g->exps, e);
    2673             : 
    2674          12 :                                         if (ne)
    2675          11 :                                                 ne = list_find_exp( gbe, ne);
    2676          12 :                                         if (ne && !list_find_exp(used, ne)) {
    2677           6 :                                                 fnd++;
    2678           6 :                                                 list_append(used, ne);
    2679             :                                         }
    2680          12 :                                         if (!ne)
    2681             :                                                 fnd = 0;
    2682             :                                 }
    2683             :                         }
    2684          12 :                         if (fnd == (list_length(gbe)+1)) {
    2685           0 :                                 v->changes++;
    2686           0 :                                 set_nodistinct(rel);
    2687             :                         }
    2688             :                 }
    2689             :         }
    2690     1232830 :         if (rel->op == op_project && rel->l &&
    2691        7044 :             need_distinct(rel) && exps_card(rel->exps) > CARD_ATOM) {
    2692             :                 node *n;
    2693        7044 :                 list *exps = new_exp_list(v->sql->sa), *gbe = new_exp_list(v->sql->sa);
    2694        7044 :                 list *obe = rel->r; /* we need to read the ordering later */
    2695             : 
    2696        7044 :                 if (obe) {
    2697             :                         int fnd = 0;
    2698             : 
    2699           0 :                         for(n = obe->h; n && !fnd; n = n->next) {
    2700           0 :                                 sql_exp *e = n->data;
    2701             : 
    2702           0 :                                 if (e->type != e_column)
    2703             :                                         fnd = 1;
    2704           0 :                                 else if (exps_bind_column2(rel->exps, e->l, e->r, NULL) == 0)
    2705             :                                         fnd = 1;
    2706             :                         }
    2707           0 :                         if (fnd)
    2708             :                                 return rel;
    2709             :                 }
    2710        7044 :                 rel->l = rel_project(v->sql->sa, rel->l, rel->exps);
    2711             : 
    2712       20876 :                 for (n = rel->exps->h; n; n = n->next) {
    2713       13832 :                         sql_exp *e = n->data, *ne;
    2714             : 
    2715       13832 :                         ne = exp_ref(v->sql, e);
    2716       13832 :                         if (e->card > CARD_ATOM && !list_find_exp(gbe, ne)) /* no need to group by on constants, or the same column multiple times */
    2717       13408 :                                 append(gbe, ne);
    2718       13832 :                         append(exps, ne);
    2719             :                 }
    2720        7044 :                 rel->op = op_groupby;
    2721        7044 :                 rel->exps = exps;
    2722        7044 :                 rel->r = gbe;
    2723        7044 :                 set_nodistinct(rel);
    2724        7044 :                 if (obe) {
    2725             :                         /* add order again */
    2726           0 :                         rel = rel_project(v->sql->sa, rel, rel_projections(v->sql, rel, NULL, 1, 1));
    2727           0 :                         rel->r = obe;
    2728             :                 }
    2729        7044 :                 v->changes++;
    2730             :         }
    2731             :         return rel;
    2732             : }
    2733             : 
    2734             : static bool exp_shares_exps(sql_exp *e, list *shared, uint64_t *uses);
    2735             : 
    2736             : static bool
    2737       97346 : exps_shares_exps(list *exps, list *shared, uint64_t *uses)
    2738             : {
    2739       97346 :         if (!exps || !shared)
    2740             :                 return false;
    2741      313941 :         for (node *n = exps->h; n; n = n->next) {
    2742      216956 :                 sql_exp *e = n->data;
    2743             : 
    2744      216956 :                 if (exp_shares_exps(e, shared, uses))
    2745             :                         return true;
    2746             :         }
    2747             :         return false;
    2748             : }
    2749             : 
    2750             : static bool
    2751     2598174 : exp_shares_exps(sql_exp *e, list *shared, uint64_t *uses)
    2752             : {
    2753     4927659 :         switch(e->type) {
    2754          13 :         case e_cmp:
    2755          13 :                 if (e->flag == cmp_or || e->flag == cmp_filter)
    2756           0 :                         return exps_shares_exps(e->l, shared, uses) || exps_shares_exps(e->r, shared, uses);
    2757          13 :                 else if (e->flag == cmp_in || e->flag == cmp_notin)
    2758           0 :                         return exp_shares_exps(e->l, shared, uses) || exps_shares_exps(e->r, shared, uses);
    2759             :                 else
    2760          13 :                         return exp_shares_exps(e->l, shared, uses) || exp_shares_exps(e->r, shared, uses) || (e->f && exp_shares_exps(e->f, shared, uses));
    2761       76478 :         case e_atom:
    2762       76478 :                 if (e->f)
    2763           0 :                         return exps_shares_exps(e->f, shared, uses);
    2764             :                 return false;
    2765     4504011 :         case e_column:
    2766             :                 {
    2767             :                         sql_exp *ne = NULL;
    2768     4504011 :                         if (e->l)
    2769     4345429 :                                 ne = exps_bind_column2(shared, e->l, e->r, NULL);
    2770     4504011 :                         if (!ne && !e->l)
    2771      158582 :                                 ne = exps_bind_column(shared, e->r, NULL, NULL, 1);
    2772     4504011 :                         if (!ne)
    2773             :                                 return false;
    2774     3520573 :                         if (ne->type != e_column) {
    2775      344610 :                                 int i = list_position(shared, ne);
    2776      344610 :                                 if (i < 0)
    2777             :                                         return false;
    2778      344610 :                                 uint64_t used = (uint64_t) 1 << i;
    2779      344610 :                                 if (used & *uses)
    2780             :                                         return true;
    2781      344298 :                                 *uses |= used;
    2782      344298 :                                 return false;
    2783             :                         }
    2784     3175963 :                         if (ne != e && (list_position(shared, e) < 0 || list_position(shared, e) > list_position(shared, ne)))
    2785             :                                 /* maybe ne refers to a local complex exp */
    2786     2079674 :                                 return exp_shares_exps(ne, shared, uses);
    2787             :                         return false;
    2788             :                 }
    2789      249811 :         case e_convert:
    2790      249811 :                 return exp_shares_exps(e->l, shared, uses);
    2791       97346 :         case e_aggr:
    2792             :         case e_func:
    2793       97346 :                 return exps_shares_exps(e->l, shared, uses);
    2794             :         case e_psm:
    2795           0 :                 assert(0);  /* not in projection list */
    2796             :         }
    2797             :         return false;
    2798             : }
    2799             : 
    2800             : static bool
    2801      284547 : exps_share_expensive_exp(list *exps, list *shared )
    2802             : {
    2803      284547 :         uint64_t uses = 0;
    2804             : 
    2805      284547 :         if (!exps || !shared)
    2806             :                 return false;
    2807     2665424 :         for (node *n = exps->h; n; n = n->next) {
    2808     2381189 :                 sql_exp *e = n->data;
    2809             : 
    2810     2381189 :                 if (exp_shares_exps(e, shared, &uses))
    2811             :                         return true;
    2812             :         }
    2813             :         return false;
    2814             : }
    2815             : 
    2816             : static bool ambigious_ref( list *exps, sql_exp *e);
    2817             : static bool
    2818       85670 : ambigious_refs( list *exps, list *refs)
    2819             : {
    2820             :         node *n;
    2821             : 
    2822       85670 :         if (!refs)
    2823             :                 return false;
    2824      249973 :         for(n=refs->h; n; n = n->next) {
    2825      181739 :                 if (ambigious_ref(exps, n->data))
    2826             :                         return true;
    2827             :         }
    2828             :         return false;
    2829             : }
    2830             : 
    2831             : static bool
    2832     3864473 : ambigious_ref( list *exps, sql_exp *e)
    2833             : {
    2834             :         sql_exp *ne = NULL;
    2835             : 
    2836     3864473 :         if (e->type == e_column) {
    2837     3428394 :                 if (e->l)
    2838     3336725 :                         ne = exps_bind_column2(exps, e->l, e->r, NULL);
    2839     3428394 :                 if (!ne && !e->l)
    2840       91669 :                         ne = exps_bind_column(exps, e->r, NULL, NULL, 1);
    2841     3428394 :                 if (ne && e != ne)
    2842             :                         return true;
    2843             :         }
    2844     3860033 :         if (e->type == e_func)
    2845       85670 :                 return ambigious_refs(exps, e->l);
    2846             :         return false;
    2847             : }
    2848             : 
    2849             : /* merge 2 projects into the lower one */
    2850             : static sql_rel *
    2851     5780578 : rel_merge_projects(visitor *v, sql_rel *rel)
    2852             : {
    2853     5846819 :         list *exps = rel->exps;
    2854     5846819 :         sql_rel *prj = rel->l;
    2855             :         node *n;
    2856             : 
    2857     5846819 :         if (rel->op == op_project &&
    2858     2021713 :             prj && prj->op == op_project && !(rel_is_ref(prj)) && !prj->r) {
    2859             :                 int all = 1;
    2860             : 
    2861      723789 :                 if (project_unsafe(rel,0) || project_unsafe(prj,0) || exps_share_expensive_exp(rel->exps, prj->exps))
    2862      439559 :                         return rel;
    2863             : 
    2864             :                 /* here we need to fix aliases */
    2865      284235 :                 rel->exps = new_exp_list(v->sql->sa);
    2866             : 
    2867             :                 /* for each exp check if we can rename it */
    2868     2230152 :                 for (n = exps->h; n && all; n = n->next) {
    2869     1950357 :                         sql_exp *e = n->data, *ne = NULL;
    2870             : 
    2871             :                         /* We do not handle expressions pointing back in the list */
    2872     1950357 :                         if (ambigious_ref(exps, e)) {
    2873             :                                 all = 0;
    2874             :                                 break;
    2875             :                         }
    2876     1945931 :                         ne = exp_push_down_prj(v->sql, e, prj, prj->l);
    2877             :                         /* check if the refered alias name isn't used twice */
    2878     1945931 :                         if (ne && ambigious_ref(rel->exps, ne)) {
    2879             :                                 all = 0;
    2880             :                                 break;
    2881             :                         }
    2882     1945917 :                         if (ne) {
    2883     1732363 :                                 if (exp_name(e))
    2884     1727535 :                                         exp_prop_alias(v->sql->sa, ne, e);
    2885     1732363 :                                 list_append(rel->exps, ne);
    2886             :                         } else {
    2887             :                                 all = 0;
    2888             :                         }
    2889             :                 }
    2890      284235 :                 if (all) {
    2891             :                         /* we can now remove the intermediate project */
    2892             :                         /* push order by expressions */
    2893       66241 :                         if (rel->r) {
    2894           0 :                                 list *nr = new_exp_list(v->sql->sa), *res = rel->r;
    2895           0 :                                 for (n = res->h; n; n = n->next) {
    2896           0 :                                         sql_exp *e = n->data, *ne = NULL;
    2897             : 
    2898           0 :                                         ne = exp_push_down_prj(v->sql, e, prj, prj->l);
    2899           0 :                                         if (ne) {
    2900           0 :                                                 if (exp_name(e))
    2901           0 :                                                         exp_prop_alias(v->sql->sa, ne, e);
    2902           0 :                                                 list_append(nr, ne);
    2903             :                                         } else {
    2904             :                                                 all = 0;
    2905             :                                         }
    2906             :                                 }
    2907           0 :                                 if (all) {
    2908           0 :                                         rel->r = nr;
    2909             :                                 } else {
    2910             :                                         /* leave as is */
    2911           0 :                                         rel->exps = exps;
    2912           0 :                                         return rel;
    2913             :                                 }
    2914             :                         }
    2915       66241 :                         rel->l = prj->l;
    2916       66241 :                         prj->l = NULL;
    2917       66241 :                         rel_destroy(prj);
    2918       66241 :                         v->changes++;
    2919       66241 :                         return rel_merge_projects(v, rel);
    2920             :                 } else {
    2921             :                         /* leave as is */
    2922      217994 :                         rel->exps = exps;
    2923             :                 }
    2924      217994 :                 return rel;
    2925             :         }
    2926             :         return rel;
    2927             : }
    2928             : 
    2929             : static inline int
    2930       57601 : str_ends_with(const char *s, const char *suffix)
    2931             : {
    2932       57601 :         size_t slen = strlen(s), suflen = strlen(suffix);
    2933       57601 :         if (suflen > slen)
    2934             :                 return 1;
    2935        1248 :         return strncmp(s + slen - suflen, suffix, suflen);
    2936             : }
    2937             : 
    2938             : static sql_exp *
    2939      593656 : exp_simplify_math( mvc *sql, sql_exp *e, int *changes)
    2940             : {
    2941      593656 :         if (e->type == e_func || e->type == e_aggr) {
    2942      132880 :                 list *l = e->l;
    2943      132880 :                 sql_subfunc *f = e->f;
    2944             :                 node *n;
    2945             :                 sql_exp *le;
    2946             : 
    2947      132880 :                 if (list_length(l) < 1)
    2948             :                         return e;
    2949             : 
    2950             :                 /* if the function has no null semantics we can return NULL if one of the arguments is NULL */
    2951      128726 :                 if (!f->func->semantics && f->func->type != F_PROC) {
    2952      208348 :                         for (node *n = l->h ; n ; n = n->next) {
    2953      132545 :                                 sql_exp *arg = n->data;
    2954             : 
    2955      132545 :                                 if (exp_is_atom(arg) && exp_is_null(arg)) {
    2956          63 :                                         sql_exp *ne = exp_null(sql->sa, exp_subtype(e));
    2957          63 :                                         (*changes)++;
    2958          63 :                                         if (exp_name(e))
    2959          19 :                                                 exp_prop_alias(sql->sa, ne, e);
    2960          63 :                                         return ne;
    2961             :                                 }
    2962             :                         }
    2963             :                 }
    2964      128663 :                 if (!f->func->s && list_length(l) == 2 && str_ends_with(f->func->imp, "_no_nil") == 0) {
    2965         105 :                         sql_exp *le = l->h->data;
    2966         105 :                         sql_exp *re = l->h->next->data;
    2967             : 
    2968             :                         /* if "_no_nil" is in the name of the
    2969             :                          * implementation function (currently either
    2970             :                          * min_no_nil or max_no_nil), in which case we
    2971             :                          * ignore the NULL and return the other value */
    2972             : 
    2973         105 :                         if (exp_is_atom(le) && exp_is_null(le)) {
    2974           0 :                                 (*changes)++;
    2975           0 :                                 if (exp_name(e))
    2976           0 :                                         exp_prop_alias(sql->sa, re, e);
    2977           0 :                                 return re;
    2978             :                         }
    2979         105 :                         if (exp_is_atom(re) && exp_is_null(re)) {
    2980           0 :                                 (*changes)++;
    2981           0 :                                 if (exp_name(e))
    2982           0 :                                         exp_prop_alias(sql->sa, le, e);
    2983           0 :                                 return le;
    2984             :                         }
    2985             :                 }
    2986             : 
    2987      128664 :                 le = l->h->data;
    2988      128664 :                 if (!EC_COMPUTE(exp_subtype(le)->type->eclass) && exp_subtype(le)->type->eclass != EC_DEC)
    2989             :                         return e;
    2990             : 
    2991       90232 :                 if (!f->func->s && !strcmp(f->func->base.name, "sql_mul") && list_length(l) == 2) {
    2992       11082 :                         sql_exp *le = l->h->data;
    2993       11082 :                         sql_exp *re = l->h->next->data;
    2994       11082 :                         sql_subtype *et = exp_subtype(e);
    2995             : 
    2996             :                         /* 0*a = 0 */
    2997       11082 :                         if (exp_is_atom(le) && exp_is_zero(le) && exp_is_atom(re) && exp_is_not_null(re)) {
    2998           0 :                                 (*changes)++;
    2999           0 :                                 le = exp_zero(sql->sa, et);
    3000           0 :                                 if (subtype_cmp(exp_subtype(e), exp_subtype(le)) != 0)
    3001           0 :                                         le = exp_convert(sql->sa, le, exp_subtype(le), exp_subtype(e));
    3002           0 :                                 if (exp_name(e))
    3003           0 :                                         exp_prop_alias(sql->sa, le, e);
    3004           0 :                                 return le;
    3005             :                         }
    3006             :                         /* a*0 = 0 */
    3007       11082 :                         if (exp_is_atom(re) && exp_is_zero(re) && exp_is_atom(le) && exp_is_not_null(le)) {
    3008           2 :                                 (*changes)++;
    3009           2 :                                 re = exp_zero(sql->sa, et);
    3010           2 :                                 if (subtype_cmp(exp_subtype(e), exp_subtype(re)) != 0)
    3011           0 :                                         re = exp_convert(sql->sa, re, exp_subtype(re), exp_subtype(e));
    3012           2 :                                 if (exp_name(e))
    3013           1 :                                         exp_prop_alias(sql->sa, re, e);
    3014           2 :                                 return re;
    3015             :                         }
    3016             :                         /* 1*a = a
    3017             :                         if (exp_is_atom(le) && exp_is_one(le)) {
    3018             :                                 (*changes)++;
    3019             :                                 if (subtype_cmp(exp_subtype(e), exp_subtype(re)) != 0)
    3020             :                                         re = exp_convert(sql->sa, re, exp_subtype(re), exp_subtype(e));
    3021             :                                 if (exp_name(e))
    3022             :                                         exp_prop_alias(sql->sa, re, e);
    3023             :                                 return re;
    3024             :                         }
    3025             :                         */
    3026             :                         /* a*1 = a
    3027             :                         if (exp_is_atom(re) && exp_is_one(re)) {
    3028             :                                 (*changes)++;
    3029             :                                 if (subtype_cmp(exp_subtype(e), exp_subtype(le)) != 0)
    3030             :                                         le = exp_convert(sql->sa, le, exp_subtype(le), exp_subtype(e));
    3031             :                                 if (exp_name(e))
    3032             :                                         exp_prop_alias(sql->sa, le, e);
    3033             :                                 return le;
    3034             :                         }
    3035             :                         */
    3036       11080 :                         if (exp_is_atom(le) && exp_is_atom(re)) {
    3037          86 :                                 atom *la = exp_flatten(sql, le);
    3038          86 :                                 atom *ra = exp_flatten(sql, re);
    3039             : 
    3040          86 :                                 if (la && ra && subtype_cmp(atom_type(la), atom_type(ra)) == 0 && subtype_cmp(atom_type(la), exp_subtype(e)) == 0) {
    3041           3 :                                         atom *a = atom_mul(la, ra);
    3042             : 
    3043           3 :                                         if (a && atom_cast(sql->sa, a, exp_subtype(e))) {
    3044           2 :                                                 sql_exp *ne = exp_atom(sql->sa, a);
    3045           2 :                                                 if (subtype_cmp(exp_subtype(e), exp_subtype(ne)) != 0)
    3046           0 :                                                         ne = exp_convert(sql->sa, ne, exp_subtype(ne), exp_subtype(e));
    3047           2 :                                                 (*changes)++;
    3048           2 :                                                 if (exp_name(e))
    3049           0 :                                                         exp_prop_alias(sql->sa, ne, e);
    3050           2 :                                                 return ne;
    3051             :                                         }
    3052             :                                 }
    3053             :                         }
    3054             :                         /* change a*a into pow(a,2), later change pow(a,2) back into a*a */
    3055             :                         if (/* DISABLES CODE */ (0) && exp_equal(le, re)==0 && exp_subtype(le)->type->eclass == EC_FLT) {
    3056             :                                 /* pow */
    3057             :                                 list *l;
    3058             :                                 sql_exp *ne;
    3059             :                                 sql_subfunc *pow = sql_bind_func(sql, "sys", "power", exp_subtype(le), exp_subtype(re), F_FUNC);
    3060             :                                 assert(pow);
    3061             :                                 if (exp_subtype(le)->type->localtype == TYPE_flt)
    3062             :                                         re = exp_atom_flt(sql->sa, 2);
    3063             :                                 else
    3064             :                                         re = exp_atom_dbl(sql->sa, 2);
    3065             :                                 l = sa_list(sql->sa);
    3066             :                                 append(l, le);
    3067             :                                 append(l, re);
    3068             :                                 (*changes)++;
    3069             :                                 ne = exp_op(sql->sa, l, pow);
    3070             :                                 if (subtype_cmp(exp_subtype(e), exp_subtype(ne)) != 0)
    3071             :                                         ne = exp_convert(sql->sa, ne, exp_subtype(ne), exp_subtype(e));
    3072             :                                 if (exp_name(e))
    3073             :                                         exp_prop_alias(sql->sa, ne, e);
    3074             :                                 return ne;
    3075             :                         }
    3076             :                         /* change a*pow(a,n) or pow(a,n)*a into pow(a,n+1) */
    3077       11078 :                         if (is_func(le->type)) {
    3078          93 :                                 list *l = le->l;
    3079          93 :                                 sql_subfunc *f = le->f;
    3080             : 
    3081          93 :                                 if (!f->func->s && !strcmp(f->func->base.name, "power") && list_length(l) == 2) {
    3082           0 :                                         sql_exp *lle = l->h->data;
    3083           0 :                                         sql_exp *lre = l->h->next->data;
    3084           0 :                                         if (exp_equal(re, lle)==0) {
    3085           0 :                                                 if (atom_inc(exp_value(sql, lre))) {
    3086           0 :                                                         if (subtype_cmp(exp_subtype(e), exp_subtype(le)) != 0)
    3087           0 :                                                                 le = exp_convert(sql->sa, le, exp_subtype(le), exp_subtype(e));
    3088           0 :                                                         (*changes)++;
    3089           0 :                                                         if (exp_name(e))
    3090           0 :                                                                 exp_prop_alias(sql->sa, le, e);
    3091           0 :                                                         return le;
    3092             :                                                 }
    3093             :                                         }
    3094             :                                 }
    3095          93 :                                 if (!f->func->s && !strcmp(f->func->base.name, "sql_mul") && list_length(l) == 2) {
    3096          30 :                                         sql_exp *lle = l->h->data;
    3097          30 :                                         sql_exp *lre = l->h->next->data;
    3098          30 :                                         if (!exp_is_atom(lle) && exp_is_atom(lre) && exp_is_atom(re)) {
    3099             :                                                 /* (x*c1)*c2 -> x * (c1*c2) */
    3100             :                                                 sql_exp *ne = NULL;
    3101             : 
    3102          13 :                                                 if (!(le = rel_binop_(sql, NULL, lre, re, "sys", "sql_mul", card_value))) {
    3103           0 :                                                         sql->session->status = 0;
    3104           0 :                                                         sql->errstr[0] = '\0';
    3105           0 :                                                         return e; /* error, fallback to original expression */
    3106             :                                                 }
    3107          13 :                                                 if (!(ne = rel_binop_(sql, NULL, lle, le, "sys", "sql_mul", card_value))) {
    3108           0 :                                                         sql->session->status = 0;
    3109           0 :                                                         sql->errstr[0] = '\0';
    3110           0 :                                                         return e; /* error, fallback to original expression */
    3111             :                                                 }
    3112          13 :                                                 if (subtype_cmp(exp_subtype(e), exp_subtype(ne)) != 0)
    3113           2 :                                                         ne = exp_convert(sql->sa, ne, exp_subtype(ne), exp_subtype(e));
    3114          13 :                                                 (*changes)++;
    3115          13 :                                                 if (exp_name(e))
    3116           8 :                                                         exp_prop_alias(sql->sa, ne, e);
    3117          13 :                                                 return ne;
    3118             :                                         }
    3119             :                                 }
    3120             :                         }
    3121             :                 }
    3122       90215 :                 if (!f->func->s && !strcmp(f->func->base.name, "sql_add") && list_length(l) == 2) {
    3123       22309 :                         sql_exp *le = l->h->data;
    3124       22309 :                         sql_exp *re = l->h->next->data;
    3125       22309 :                         if (exp_is_atom(le) && exp_is_zero(le)) {
    3126           0 :                                 if (subtype_cmp(exp_subtype(e), exp_subtype(re)) != 0)
    3127           0 :                                         re = exp_convert(sql->sa, re, exp_subtype(re), exp_subtype(e));
    3128           0 :                                 (*changes)++;
    3129           0 :                                 if (exp_name(e))
    3130           0 :                                         exp_prop_alias(sql->sa, re, e);
    3131           0 :                                 return re;
    3132             :                         }
    3133       22309 :                         if (exp_is_atom(re) && exp_is_zero(re)) {
    3134           4 :                                 if (subtype_cmp(exp_subtype(e), exp_subtype(le)) != 0)
    3135           0 :                                         le = exp_convert(sql->sa, le, exp_subtype(le), exp_subtype(e));
    3136           4 :                                 (*changes)++;
    3137           4 :                                 if (exp_name(e))
    3138           4 :                                         exp_prop_alias(sql->sa, le, e);
    3139           4 :                                 return le;
    3140             :                         }
    3141       22305 :                         if (exp_is_atom(le) && exp_is_atom(re)) {
    3142          94 :                                 atom *la = exp_flatten(sql, le);
    3143          94 :                                 atom *ra = exp_flatten(sql, re);
    3144             : 
    3145          94 :                                 if (la && ra) {
    3146          18 :                                         atom *a = atom_add(la, ra);
    3147             : 
    3148          18 :                                         if (a) {
    3149          17 :                                                 sql_exp *ne = exp_atom(sql->sa, a);
    3150          17 :                                                 if (subtype_cmp(exp_subtype(e), exp_subtype(ne)) != 0)
    3151           2 :                                                         ne = exp_convert(sql->sa, ne, exp_subtype(ne), exp_subtype(e));
    3152          17 :                                                 (*changes)++;
    3153          17 :                                                 if (exp_name(e))
    3154           2 :                                                         exp_prop_alias(sql->sa, ne, e);
    3155          17 :                                                 return ne;
    3156             :                                         }
    3157             :                                 }
    3158             :                         }
    3159       22288 :                         if (is_func(le->type)) {
    3160        2939 :                                 list *ll = le->l;
    3161        2939 :                                 sql_subfunc *f = le->f;
    3162        2939 :                                 if (!f->func->s && !strcmp(f->func->base.name, "sql_add") && list_length(ll) == 2) {
    3163        2502 :                                         sql_exp *lle = ll->h->data;
    3164        2502 :                                         sql_exp *lre = ll->h->next->data;
    3165             : 
    3166        2502 :                                         if (exp_is_atom(lle) && exp_is_atom(lre))
    3167             :                                                 return e;
    3168        2491 :                                         if (!exp_is_atom(re) && exp_is_atom(lre)) {
    3169             :                                                 /* (x+c1)+y -> (x+y) + c1 */
    3170           1 :                                                 ll->h->next->data = re;
    3171           1 :                                                 l->h->next->data = lre;
    3172           1 :                                                 if (!(l->h->data = exp_simplify_math(sql, le, changes)))
    3173             :                                                         return NULL;
    3174           1 :                                                 (*changes)++;
    3175           1 :                                                 return e;
    3176             :                                         }
    3177        2490 :                                         if (exp_is_atom(re) && exp_is_atom(lre)) {
    3178             :                                                 /* (x+c1)+c2 -> (c2+c1) + x */
    3179           3 :                                                 ll->h->data = re;
    3180           3 :                                                 l->h->next->data = lle;
    3181           3 :                                                 if (!(l->h->data = exp_simplify_math(sql, le, changes)))
    3182             :                                                         return NULL;
    3183           3 :                                                 (*changes)++;
    3184           3 :                                                 return e;
    3185             :                                         }
    3186             :                                 }
    3187             :                         }
    3188             :                         /*
    3189             :                         if (is_func(re->type)) {
    3190             :                                 list *ll = re->l;
    3191             :                                 sql_subfunc *f = re->f;
    3192             :                                 if (!f->func->s && !strcmp(f->func->base.name, "sql_add") && list_length(ll) == 2) {
    3193             :                                         if (exp_is_atom(le)) {
    3194             :                                                 * c1+(x+y) -> (x+y) + c1 *
    3195             :                                                 l->h->data = re;
    3196             :                                                 l->h->next->data = le;
    3197             :                                                 (*changes)++;
    3198             :                                                 return e;
    3199             :                                         }
    3200             :                                 }
    3201             :                         }
    3202             :                         */
    3203             :                 }
    3204       90179 :                 if (!f->func->s && !strcmp(f->func->base.name, "sql_sub") && list_length(l) == 2) {
    3205        5425 :                         sql_exp *le = l->h->data;
    3206        5425 :                         sql_exp *re = l->h->next->data;
    3207             : 
    3208        5425 :                         if (exp_is_atom(le) && exp_is_atom(re)) {
    3209          25 :                                 atom *la = exp_flatten(sql, le);
    3210          25 :                                 atom *ra = exp_flatten(sql, re);
    3211             : 
    3212          25 :                                 if (la && ra) {
    3213           5 :                                         atom *a = atom_sub(la, ra);
    3214             : 
    3215           5 :                                         if (a) {
    3216           4 :                                                 sql_exp *ne = exp_atom(sql->sa, a);
    3217           4 :                                                 if (subtype_cmp(exp_subtype(e), exp_subtype(ne)) != 0)
    3218           3 :                                                         ne = exp_convert(sql->sa, ne, exp_subtype(ne), exp_subtype(e));
    3219           4 :                                                 (*changes)++;
    3220           4 :                                                 if (exp_name(e))
    3221           0 :                                                         exp_prop_alias(sql->sa, ne, e);
    3222           4 :                                                 return ne;
    3223             :                                         }
    3224             :                                 }
    3225             :                         }
    3226        5421 :                         if (exp_is_not_null(le) && exp_is_not_null(re) && exp_equal(le,re) == 0) { /* a - a = 0 */
    3227             :                                 atom *a;
    3228             :                                 sql_exp *ne;
    3229             : 
    3230           0 :                                 if (exp_subtype(le)->type->eclass == EC_NUM) {
    3231           0 :                                         a = atom_int(sql->sa, exp_subtype(le), 0);
    3232           0 :                                 } else if (exp_subtype(le)->type->eclass == EC_FLT) {
    3233           0 :                                         a = atom_float(sql->sa, exp_subtype(le), 0);
    3234             :                                 } else {
    3235             :                                         return e;
    3236             :                                 }
    3237           0 :                                 ne = exp_atom(sql->sa, a);
    3238           0 :                                 if (subtype_cmp(exp_subtype(e), exp_subtype(ne)) != 0)
    3239           0 :                                         ne = exp_convert(sql->sa, ne, exp_subtype(ne), exp_subtype(e));
    3240           0 :                                 (*changes)++;
    3241           0 :                                 if (exp_name(e))
    3242           0 :                                         exp_prop_alias(sql->sa, ne, e);
    3243           0 :                                 return ne;
    3244             :                         }
    3245        5421 :                         if (is_func(le->type)) {
    3246          47 :                                 list *ll = le->l;
    3247          47 :                                 sql_subfunc *f = le->f;
    3248          47 :                                 if (!f->func->s && !strcmp(f->func->base.name, "sql_add") && list_length(ll) == 2) {
    3249           7 :                                         sql_exp *lle = ll->h->data;
    3250           7 :                                         sql_exp *lre = ll->h->next->data;
    3251           7 :                                         if (exp_equal(re, lre) == 0) {
    3252             :                                                 /* (x+a)-a = x*/
    3253           0 :                                                 if (subtype_cmp(exp_subtype(e), exp_subtype(lle)) != 0)
    3254           0 :                                                         lle = exp_convert(sql->sa, lle, exp_subtype(lle), exp_subtype(e));
    3255           0 :                                                 if (exp_name(e))
    3256           0 :                                                         exp_prop_alias(sql->sa, lle, e);
    3257           0 :                                                 (*changes)++;
    3258           0 :                                                 return lle;
    3259             :                                         }
    3260           7 :                                         if (exp_is_atom(lle) && exp_is_atom(lre))
    3261             :                                                 return e;
    3262           7 :                                         if (!exp_is_atom(re) && exp_is_atom(lre)) {
    3263             :                                                 /* (x+c1)-y -> (x-y) + c1 */
    3264           3 :                                                 ll->h->next->data = re;
    3265           3 :                                                 l->h->next->data = lre;
    3266           3 :                                                 le->f = e->f;
    3267           3 :                                                 e->f = f;
    3268           3 :                                                 if (!(l->h->data = exp_simplify_math(sql, le, changes)))
    3269             :                                                         return NULL;
    3270           3 :                                                 (*changes)++;
    3271           3 :                                                 return e;
    3272             :                                         }
    3273           4 :                                         if (exp_is_atom(re) && exp_is_atom(lre)) {
    3274             :                                                 /* (x+c1)-c2 -> (c1-c2) + x */
    3275           0 :                                                 ll->h->data = lre;
    3276           0 :                                                 ll->h->next->data = re;
    3277           0 :                                                 l->h->next->data = lle;
    3278           0 :                                                 le->f = e->f;
    3279           0 :                                                 e->f = f;
    3280           0 :                                                 if (!(l->h->data = exp_simplify_math(sql, le, changes)))
    3281             :                                                         return NULL;
    3282           0 :                                                 (*changes)++;
    3283           0 :                                                 return e;
    3284             :                                         }
    3285             :                                 }
    3286             :                         }
    3287             :                 }
    3288             :                 if (l)
    3289      260939 :                         for (n = l->h; n; n = n->next)
    3290      170767 :                                 if (!(n->data = exp_simplify_math(sql, n->data, changes)))
    3291             :                                         return NULL;
    3292             :         }
    3293      550948 :         if (e->type == e_convert)
    3294       68002 :                 if (!(e->l = exp_simplify_math(sql, e->l, changes)))
    3295           0 :                         return NULL;
    3296             :         return e;
    3297             : }
    3298             : 
    3299             : static inline sql_rel *
    3300     1272616 : rel_simplify_math(visitor *v, sql_rel *rel)
    3301             : {
    3302     1272616 :         if ((is_simple_project(rel->op) || is_groupby(rel->op) || (rel->op == op_ddl && rel->flag == ddl_psm)) && rel->exps) {
    3303      346965 :                 int needed = 0, ochanges = 0;
    3304             : 
    3305     1635967 :                 for (node *n = rel->exps->h; n && !needed; n = n->next) {
    3306     1289002 :                         sql_exp *e = n->data;
    3307             : 
    3308     1289002 :                         if (e->type == e_func || e->type == e_convert || e->type == e_aggr || e->type == e_psm)
    3309             :                                 needed = 1;
    3310             :                 }
    3311      346965 :                 if (!needed)
    3312      271118 :                         return rel;
    3313             : 
    3314      430728 :                 for (node *n = rel->exps->h; n; n = n->next) {
    3315      354880 :                         sql_exp *ne = exp_simplify_math(v->sql, n->data, &ochanges);
    3316             : 
    3317      354881 :                         if (!ne)
    3318             :                                 return NULL;
    3319      354881 :                         n->data = ne;
    3320             :                 }
    3321       75848 :                 v->changes += ochanges;
    3322             :         }
    3323             :         return rel;
    3324             : }
    3325             : 
    3326             : 
    3327             : static sql_rel *
    3328      124387 : rel_find_ref( sql_rel *r)
    3329             : {
    3330    11601205 :         while (!rel_is_ref(r) && r->l &&
    3331    11593054 :               (is_project(r->op) || is_select(r->op) /*|| is_join(r->op)*/))
    3332             :                 r = r->l;
    3333      124387 :         if (rel_is_ref(r))
    3334        6832 :                 return r;
    3335             :         return NULL;
    3336             : }
    3337             : 
    3338             : static sql_rel *
    3339        4648 : rel_find_select( sql_rel *r)
    3340             : {
    3341       15304 :         while (!is_select(r->op) && r->l && is_project(r->op))
    3342             :                 r = r->l;
    3343        4648 :         if (is_select(r->op))
    3344         338 :                 return r;
    3345             :         return NULL;
    3346             : }
    3347             : 
    3348             : static int
    3349         169 : rel_match_projections(sql_rel *l, sql_rel *r)
    3350             : {
    3351             :         node *n, *m;
    3352         169 :         list *le = l->exps;
    3353         169 :         list *re = r->exps;
    3354             : 
    3355         169 :         if (!le || !re)
    3356             :                 return 0;
    3357         169 :         if (list_length(le) != list_length(re))
    3358             :                 return 0;
    3359             : 
    3360         948 :         for (n = le->h, m = re->h; n && m; n = n->next, m = m->next)
    3361         820 :                 if (!exp_match(n->data, m->data))
    3362             :                         return 0;
    3363             :         return 1;
    3364             : }
    3365             : 
    3366             : static int
    3367             : exps_has_predicate( list *l )
    3368             : {
    3369             :         node *n;
    3370             : 
    3371           0 :         for( n = l->h; n; n = n->next){
    3372           0 :                 sql_exp *e = n->data;
    3373             : 
    3374           0 :                 if (e->card <= CARD_ATOM)
    3375             :                         return 1;
    3376             :         }
    3377             :         return 0;
    3378             : }
    3379             : 
    3380             : static inline sql_rel *
    3381     3868468 : rel_merge_union(visitor *v, sql_rel *rel)
    3382             : {
    3383     3868468 :         sql_rel *l = rel->l;
    3384     3868468 :         sql_rel *r = rel->r;
    3385             :         sql_rel *ref = NULL;
    3386             : 
    3387     3868468 :         if (is_union(rel->op) &&
    3388      213444 :             l && is_project(l->op) && !project_unsafe(l,0) &&
    3389      243899 :             r && is_project(r->op) && !project_unsafe(r,0) &&
    3390      124386 :             (ref = rel_find_ref(l)) != NULL && ref == rel_find_ref(r)) {
    3391             :                 /* Find selects and try to merge */
    3392        2324 :                 sql_rel *ls = rel_find_select(l);
    3393        2324 :                 sql_rel *rs = rel_find_select(r);
    3394             : 
    3395             :                 /* can we merge ? */
    3396        2324 :                 if (!ls || !rs)
    3397             :                         return rel;
    3398             : 
    3399             :                 /* merge any extra projects */
    3400         169 :                 if (l->l != ls)
    3401         147 :                         rel->l = l = rel_merge_projects(v, l);
    3402         169 :                 if (r->l != rs)
    3403          21 :                         rel->r = r = rel_merge_projects(v, r);
    3404             : 
    3405         169 :                 if (!rel_match_projections(l,r))
    3406             :                         return rel;
    3407             : 
    3408             :                 /* for now only union(project*(select(R),project*(select(R))) */
    3409         128 :                 if (ls != l->l || rs != r->l ||
    3410           0 :                     ls->l != rs->l || !rel_is_ref(ls->l))
    3411             :                         return rel;
    3412             : 
    3413           0 :                 if (!ls->exps || !rs->exps ||
    3414           0 :                     exps_has_predicate(ls->exps) ||
    3415             :                     exps_has_predicate(rs->exps))
    3416             :                         return rel;
    3417             : 
    3418             :                 /* merge, ie. add 'or exp' */
    3419           0 :                 v->changes++;
    3420           0 :                 ls->exps = append(new_exp_list(v->sql->sa), exp_or(v->sql->sa, ls->exps, rs->exps, 0));
    3421           0 :                 rs->exps = NULL;
    3422           0 :                 rel = rel_inplace_project(v->sql->sa, rel, rel_dup(rel->l), rel->exps);
    3423           0 :                 set_processed(rel);
    3424           0 :                 return rel;
    3425             :         }
    3426             :         return rel;
    3427             : }
    3428             : 
    3429             : static int
    3430       61052 : exps_cse( mvc *sql, list *oexps, list *l, list *r )
    3431             : {
    3432             :         list *nexps;
    3433             :         node *n, *m;
    3434             :         char *lu, *ru;
    3435             :         int lc = 0, rc = 0, match = 0, res = 0;
    3436             : 
    3437       61052 :         if (list_length(l) == 0 || list_length(r) == 0)
    3438           0 :                 return 0;
    3439             : 
    3440             :         /* first recusive exps_cse */
    3441       61052 :         nexps = new_exp_list(sql->sa);
    3442      126879 :         for (n = l->h; n; n = n->next) {
    3443       65827 :                 sql_exp *e = n->data;
    3444             : 
    3445       65827 :                 if (e->type == e_cmp && e->flag == cmp_or && !is_anti(e)) {
    3446       14718 :                         res = exps_cse(sql, nexps, e->l, e->r);
    3447             :                 } else {
    3448       51109 :                         append(nexps, e);
    3449             :                 }
    3450             :         }
    3451             :         l = nexps;
    3452             : 
    3453       61052 :         nexps = new_exp_list(sql->sa);
    3454      128898 :         for (n = r->h; n; n = n->next) {
    3455       67846 :                 sql_exp *e = n->data;
    3456             : 
    3457       67846 :                 if (e->type == e_cmp && e->flag == cmp_or && !is_anti(e)) {
    3458        3082 :                         res = exps_cse(sql, nexps, e->l, e->r);
    3459             :                 } else {
    3460       64764 :                         append(nexps, e);
    3461             :                 }
    3462             :         }
    3463             :         r = nexps;
    3464             : 
    3465             :         /* simplify  true or .. and .. or true */
    3466       61052 :         if (list_length(l) == list_length(r) && list_length(l) == 1) {
    3467       56425 :                 sql_exp *le = l->h->data, *re = r->h->data;
    3468             : 
    3469       56425 :                 if (exp_is_true(le)) {
    3470           5 :                         append(oexps, le);
    3471           5 :                         return 1;
    3472             :                 }
    3473       56420 :                 if (exp_is_true(re)) {
    3474           0 :                         append(oexps, re);
    3475           0 :                         return 1;
    3476             :                 }
    3477             :         }
    3478             : 
    3479       61047 :         lu = SA_ZNEW_ARRAY(sql->ta, char, list_length(l));
    3480       61047 :         ru = SA_ZNEW_ARRAY(sql->ta, char, list_length(r));
    3481      126892 :         for (n = l->h, lc = 0; n; n = n->next, lc++) {
    3482       65845 :                 sql_exp *le = n->data;
    3483             : 
    3484      141262 :                 for ( m = r->h, rc = 0; m; m = m->next, rc++) {
    3485       75417 :                         sql_exp *re = m->data;
    3486             : 
    3487       75417 :                         if (!ru[rc] && exp_match_exp(le,re)) {
    3488          73 :                                 lu[lc] = 1;
    3489          73 :                                 ru[rc] = 1;
    3490             :                                 match = 1;
    3491             :                         }
    3492             :                 }
    3493             :         }
    3494       61047 :         if (match) {
    3495          51 :                 list *nl = new_exp_list(sql->sa);
    3496          51 :                 list *nr = new_exp_list(sql->sa);
    3497             : 
    3498         181 :                 for (n = l->h, lc = 0; n; n = n->next, lc++)
    3499         130 :                         if (!lu[lc])
    3500          60 :                                 append(nl, n->data);
    3501         191 :                 for (n = r->h, rc = 0; n; n = n->next, rc++)
    3502         140 :                         if (!ru[rc])
    3503          67 :                                 append(nr, n->data);
    3504             : 
    3505          51 :                 if (list_length(nl) && list_length(nr))
    3506          28 :                         append(oexps, exp_or(sql->sa, nl, nr, 0));
    3507             : 
    3508         181 :                 for (n = l->h, lc = 0; n; n = n->next, lc++) {
    3509         130 :                         if (lu[lc])
    3510          70 :                                 append(oexps, n->data);
    3511             :                 }
    3512             :                 res = 1;
    3513             :         } else {
    3514       60996 :                 append(oexps, exp_or(sql->sa, list_dup(l, (fdup)NULL),
    3515             :                                      list_dup(r, (fdup)NULL), 0));
    3516             :         }
    3517             :         return res;
    3518             : }
    3519             : 
    3520             : static int
    3521       99666 : are_equality_exps( list *exps, sql_exp **L)
    3522             : {
    3523       99666 :         sql_exp *l = *L;
    3524             : 
    3525       99666 :         if (list_length(exps) == 1) {
    3526       95925 :                 sql_exp *e = exps->h->data, *le = e->l, *re = e->r;
    3527             : 
    3528       95925 :                 if (e->type == e_cmp && e->flag == cmp_equal && le->card != CARD_ATOM && re->card == CARD_ATOM && !is_semantics(e)) {
    3529       33346 :                         if (!l) {
    3530       22920 :                                 *L = l = le;
    3531       22920 :                                 if (!is_column(le->type))
    3532             :                                         return 0;
    3533             :                         }
    3534       33346 :                         return (exp_match(l, le));
    3535             :                 }
    3536       62579 :                 if (e->type == e_cmp && e->flag == cmp_or && !is_anti(e) && !is_semantics(e))
    3537       22617 :                         return (are_equality_exps(e->l, L) && are_equality_exps(e->r, L));
    3538             :         }
    3539             :         return 0;
    3540             : }
    3541             : 
    3542             : static void
    3543        3544 : get_exps( list *n, list *l )
    3544             : {
    3545        4534 :         sql_exp *e = l->h->data, *re = e->r;
    3546             : 
    3547        4534 :         if (e->type == e_cmp && e->flag == cmp_equal && re->card == CARD_ATOM)
    3548        3544 :                 list_append(n, re);
    3549        4534 :         if (e->type == e_cmp && e->flag == cmp_or) {
    3550         990 :                 get_exps(n, e->l);
    3551         990 :                 get_exps(n, e->r);
    3552             :         }
    3553        3544 : }
    3554             : 
    3555             : static sql_exp *
    3556        1277 : equality_exps_2_in( mvc *sql, sql_exp *ce, list *l, list *r)
    3557             : {
    3558        1277 :         list *nl = new_exp_list(sql->sa);
    3559             : 
    3560        1277 :         get_exps(nl, l);
    3561        1277 :         get_exps(nl, r);
    3562             : 
    3563        1277 :         return exp_in( sql->sa, ce, nl, cmp_in);
    3564             : }
    3565             : 
    3566             : static list *
    3567      694912 : merge_ors(mvc *sql, list *exps, int *changes)
    3568             : {
    3569             :         list *nexps = NULL;
    3570             :         int needed = 0;
    3571             : 
    3572     1526257 :         for (node *n = exps->h; n && !needed; n = n->next) {
    3573      831345 :                 sql_exp *e = n->data;
    3574             : 
    3575      831345 :                 if (e->type == e_cmp && e->flag == cmp_or && !is_anti(e))
    3576             :                         needed = 1;
    3577             :         }
    3578             : 
    3579      694912 :         if (needed) {
    3580       51653 :                 nexps = new_exp_list(sql->sa);
    3581      111956 :                 for (node *n = exps->h; n; n = n->next) {
    3582       60303 :                         sql_exp *e = n->data, *l = NULL;
    3583             : 
    3584       60303 :                         if (e->type == e_cmp && e->flag == cmp_or && !is_anti(e) && are_equality_exps(e->l, &l) && are_equality_exps(e->r, &l) && l) {
    3585        1277 :                                 (*changes)++;
    3586        1277 :                                 append(nexps, equality_exps_2_in(sql, l, e->l, e->r));
    3587             :                         } else {
    3588       59026 :                                 append(nexps, e);
    3589             :                         }
    3590             :                 }
    3591             :         } else {
    3592             :                 nexps = exps;
    3593             :         }
    3594             : 
    3595     1532551 :         for (node *n = nexps->h; n ; n = n->next) {
    3596      837639 :                 sql_exp *e = n->data;
    3597             : 
    3598      837639 :                 if (e->type == e_cmp && e->flag == cmp_or) {
    3599       51556 :                         e->l = merge_ors(sql, e->l, changes);
    3600       51556 :                         e->r = merge_ors(sql, e->r, changes);
    3601             :                 }
    3602             :         }
    3603             : 
    3604      694912 :         return nexps;
    3605             : }
    3606             : 
    3607             : #define TRIVIAL_NOT_EQUAL_CMP(e) \
    3608             :         ((e)->type == e_cmp && (e)->flag == cmp_notequal && !is_anti((e)) && !is_semantics((e)) && ((sql_exp*)(e)->l)->card != CARD_ATOM && ((sql_exp*)(e)->r)->card == CARD_ATOM)
    3609             : 
    3610             : static list *
    3611      694912 : merge_notequal(mvc *sql, list *exps, int *changes)
    3612             : {
    3613             :         list *inequality_groups = NULL, *nexps = NULL;
    3614             :         int needed = 0;
    3615             : 
    3616     1532551 :         for (node *n = exps->h; n; n = n->next) {
    3617      837639 :                 sql_exp *e = n->data;
    3618             : 
    3619      837639 :                 if (TRIVIAL_NOT_EQUAL_CMP(e)) {
    3620             :                         bool appended = false;
    3621             : 
    3622      127333 :                         if (inequality_groups) {
    3623        8367 :                                 for (node *m = inequality_groups->h; m && !appended; m = m->next) {
    3624        4805 :                                         list *next = m->data;
    3625        4805 :                                         sql_exp *first = (sql_exp*) next->h->data;
    3626             : 
    3627        4805 :                                         if (exp_match(first->l, e->l)) {
    3628         645 :                                                 list_append(next, e);
    3629             :                                                 appended = true;
    3630             :                                         }
    3631             :                                 }
    3632             :                         }
    3633        3562 :                         if (!appended) {
    3634      126688 :                                 if (!inequality_groups)
    3635      123771 :                                         inequality_groups = new_exp_list(sql->sa);
    3636      126688 :                                 list_append(inequality_groups, list_append(new_exp_list(sql->sa), e));
    3637             :                         }
    3638             :                 }
    3639             :         }
    3640             : 
    3641      694912 :         if (inequality_groups) { /* if one list of inequalities has more than one entry, then the re-write is needed */
    3642      250459 :                 for (node *n = inequality_groups->h; n; n = n->next) {
    3643      126688 :                         list *next = n->data;
    3644             : 
    3645      126688 :                         if (list_length(next) > 1)
    3646             :                                 needed = 1;
    3647             :                 }
    3648             :         }
    3649             : 
    3650      123771 :         if (needed) {
    3651         478 :                 nexps = new_exp_list(sql->sa);
    3652        1571 :                 for (node *n = inequality_groups->h; n; n = n->next) {
    3653        1093 :                         list *next = n->data;
    3654        1093 :                         sql_exp *first = (sql_exp*) next->h->data;
    3655        1093 :                         list *notin = new_exp_list(sql->sa);
    3656             : 
    3657        2831 :                         for (node *m = next->h; m; m = m->next) {
    3658        1738 :                                 sql_exp *e = m->data;
    3659        1738 :                                 list_append(notin, e->r);
    3660             :                         }
    3661        1093 :                         list_append(nexps, exp_in(sql->sa, first->l, notin, cmp_notin));
    3662             :                 }
    3663             : 
    3664        2574 :                 for (node *n = exps->h; n; n = n->next) {
    3665        2096 :                         sql_exp *e = n->data;
    3666             : 
    3667        2096 :                         if (!TRIVIAL_NOT_EQUAL_CMP(e))
    3668         358 :                                 list_append(nexps, e);
    3669             :                 }
    3670         478 :                 (*changes)++;
    3671             :         } else {
    3672             :                 nexps = exps;
    3673             :         }
    3674             : 
    3675     1531906 :         for (node *n = nexps->h; n ; n = n->next) {
    3676      836994 :                 sql_exp *e = n->data;
    3677             : 
    3678      836994 :                 if (e->type == e_cmp && e->flag == cmp_or) {
    3679       51556 :                         e->l = merge_notequal(sql, e->l, changes);
    3680       51556 :                         e->r = merge_notequal(sql, e->r, changes);
    3681             :                 }
    3682             :         }
    3683             : 
    3684      694912 :         return nexps;
    3685             : }
    3686             : 
    3687             : static int
    3688      258174 : is_numeric_upcast(sql_exp *e)
    3689             : {
    3690      258174 :         if (is_convert(e->type)) {
    3691       24927 :                 sql_subtype *f = exp_fromtype(e);
    3692       24927 :                 sql_subtype *t = exp_totype(e);
    3693             : 
    3694       24927 :                 if (f->type->eclass == t->type->eclass && EC_COMPUTE(f->type->eclass)) {
    3695        1853 :                         if (f->type->localtype < t->type->localtype)
    3696        1845 :                                 return 1;
    3697             :                 }
    3698             :         }
    3699             :         return 0;
    3700             : }
    3701             : 
    3702             : /* optimize (a = b) or (a is null and b is null) -> a = b with null semantics */
    3703             : static sql_exp *
    3704       86976 : try_rewrite_equal_or_is_null(mvc *sql, sql_rel *rel, sql_exp *or, list *l1, list *l2)
    3705             : {
    3706       86976 :         if (list_length(l1) == 1) {
    3707             :                 bool valid = true, first_is_null_found = false, second_is_null_found = false;
    3708       83608 :                 sql_exp *cmp = l1->h->data;
    3709       83608 :                 sql_exp *first = cmp->l, *second = cmp->r;
    3710             : 
    3711       83608 :                 if (is_compare(cmp->type) && !is_anti(cmp) && !cmp->f && cmp->flag == cmp_equal) {
    3712       53670 :                         int fupcast = is_numeric_upcast(first), supcast = is_numeric_upcast(second);
    3713      107578 :                         for(node *n = l2->h ; n && valid; n = n->next) {
    3714       53908 :                                 sql_exp *e = n->data, *l = e->l, *r = e->r;
    3715             : 
    3716       53908 :                                 if (is_compare(e->type) && e->flag == cmp_equal && !e->f &&
    3717       51611 :                                         !is_anti(e) && is_semantics(e)) {
    3718       25433 :                                         int lupcast = is_numeric_upcast(l);
    3719       25433 :                                         int rupcast = is_numeric_upcast(r);
    3720       25433 :                                         sql_exp *rr = rupcast ? r->l : r;
    3721             : 
    3722       25433 :                                         if (rr->type == e_atom && rr->l && atom_null(rr->l)) {
    3723       25433 :                                                 if (exp_match_exp(fupcast?first->l:first, lupcast?l->l:l))
    3724             :                                                         first_is_null_found = true;
    3725         401 :                                                 else if (exp_match_exp(supcast?second->l:second, lupcast?l->l:l))
    3726             :                                                         second_is_null_found = true;
    3727             :                                                 else
    3728             :                                                         valid = false;
    3729             :                                         } else {
    3730             :                                                 valid = false;
    3731             :                                         }
    3732             :                                 } else {
    3733             :                                         valid = false;
    3734             :                                 }
    3735             :                         }
    3736       53670 :                         if (valid && first_is_null_found && second_is_null_found) {
    3737             :                                 sql_subtype super;
    3738             : 
    3739         236 :                                 supertype(&super, exp_subtype(first), exp_subtype(second)); /* first and second must have the same type */
    3740         472 :                                 if (!(first = exp_check_type(sql, &super, rel, first, type_equal)) ||
    3741         236 :                                         !(second = exp_check_type(sql, &super, rel, second, type_equal))) {
    3742           0 :                                                 sql->session->status = 0;
    3743           0 :                                                 sql->errstr[0] = 0;
    3744           0 :                                                 return or;
    3745             :                                         }
    3746         236 :                                 sql_exp *res = exp_compare(sql->sa, first, second, cmp->flag);
    3747         236 :                                 set_semantics(res);
    3748         236 :                                 if (exp_name(or))
    3749           0 :                                         exp_prop_alias(sql->sa, res, or);
    3750         236 :                                 return res;
    3751             :                         }
    3752             :                 }
    3753             :         }
    3754             :         return or;
    3755             : }
    3756             : 
    3757             : static list *
    3758     1388108 : merge_cmp_or_null(mvc *sql, sql_rel *rel, list *exps, int *changes)
    3759             : {
    3760     3054737 :         for (node *n = exps->h; n ; n = n->next) {
    3761     1666629 :                 sql_exp *e = n->data;
    3762             : 
    3763     1666629 :                 if (is_compare(e->type) && e->flag == cmp_or && !is_anti(e)) {
    3764       43488 :                         sql_exp *ne = try_rewrite_equal_or_is_null(sql, rel, e, e->l, e->r);
    3765       43488 :                         if (ne != e) {
    3766         234 :                                 (*changes)++;
    3767         234 :                                 n->data = ne;
    3768             :                         }
    3769       43488 :                         ne = try_rewrite_equal_or_is_null(sql, rel, e, e->r, e->l);
    3770       43488 :                         if (ne != e) {
    3771           2 :                                 (*changes)++;
    3772           2 :                                 n->data = ne;
    3773             :                         }
    3774             :                 }
    3775             :         }
    3776     1388108 :         return exps;
    3777             : }
    3778             : 
    3779             : static inline sql_rel *
    3780     1388108 : rel_select_cse(visitor *v, sql_rel *rel)
    3781             : {
    3782     1388108 :         if (is_select(rel->op) && rel->exps)
    3783      591800 :                 rel->exps = merge_ors(v->sql, rel->exps, &v->changes); /* x = 1 or x = 2 => x in (1, 2)*/
    3784             : 
    3785     1388108 :         if (is_select(rel->op) && rel->exps)
    3786      591800 :                 rel->exps = merge_notequal(v->sql, rel->exps, &v->changes); /* x <> 1 and x <> 2 => x not in (1, 2)*/
    3787             : 
    3788     1388108 :         if ((is_select(rel->op) || is_join(rel->op) || is_semi(rel->op)) && rel->exps)
    3789     1388108 :                 rel->exps = merge_cmp_or_null(v->sql, rel, rel->exps, &v->changes); /* (a = b) or (a is null and b is null) -> a = b with null semantics */
    3790             : 
    3791     1388108 :         if ((is_select(rel->op) || is_join(rel->op) || is_semi(rel->op)) && rel->exps) {
    3792             :                 node *n;
    3793             :                 list *nexps;
    3794             :                 int needed = 0;
    3795             : 
    3796     3049271 :                 for (n=rel->exps->h; n && !needed; n = n->next) {
    3797     1661163 :                         sql_exp *e = n->data;
    3798             : 
    3799     1661163 :                         if (e->type == e_cmp && e->flag == cmp_or && !is_anti(e))
    3800             :                                 needed = 1;
    3801             :                 }
    3802     1388108 :                 if (!needed)
    3803             :                         return rel;
    3804       42375 :                 nexps = new_exp_list(v->sql->sa);
    3805       92902 :                 for (n=rel->exps->h; n; n = n->next) {
    3806       50527 :                         sql_exp *e = n->data;
    3807             : 
    3808       50527 :                         if (e->type == e_cmp && e->flag == cmp_or && !is_anti(e)) {
    3809             :                                 /* split the common expressions */
    3810       43252 :                                 v->changes += exps_cse(v->sql, nexps, e->l, e->r);
    3811             :                         } else {
    3812        7275 :                                 append(nexps, e);
    3813             :                         }
    3814             :                 }
    3815       42375 :                 rel->exps = nexps;
    3816             :         }
    3817             :         return rel;
    3818             : }
    3819             : 
    3820             : static inline sql_rel *
    3821     4942461 : rel_project_cse(visitor *v, sql_rel *rel)
    3822             : {
    3823     4942461 :         if (is_project(rel->op) && rel->exps) {
    3824             :                 node *n, *m;
    3825             :                 list *nexps;
    3826             :                 int needed = 0;
    3827             : 
    3828    10646589 :                 for (n=rel->exps->h; n && !needed; n = n->next) {
    3829     8712592 :                         sql_exp *e1 = n->data;
    3830             : 
    3831     8712592 :                         if (e1->type != e_column && !exp_is_atom(e1) && exp_name(e1)) {
    3832     2857833 :                                 for (m=n->next; m; m = m->next){
    3833     1973018 :                                         sql_exp *e2 = m->data;
    3834             : 
    3835     1973018 :                                         if (exp_name(e2) && exp_match_exp(e1, e2))
    3836             :                                                 needed = 1;
    3837             :                                 }
    3838             :                         }
    3839             :                 }
    3840             : 
    3841     1933997 :                 if (!needed)
    3842             :                         return rel;
    3843             : 
    3844         425 :                 nexps = new_exp_list(v->sql->sa);
    3845        5376 :                 for (n=rel->exps->h; n; n = n->next) {
    3846        4951 :                         sql_exp *e1 = n->data;
    3847             : 
    3848        4951 :                         if (e1->type != e_column && !exp_is_atom(e1) && exp_name(e1)) {
    3849       28091 :                                 for (m=nexps->h; m; m = m->next){
    3850       25043 :                                         sql_exp *e2 = m->data;
    3851             : 
    3852       25043 :                                         if (exp_name(e2) && exp_match_exp(e1, e2) && (e1->type != e_column || exps_bind_column2(nexps, exp_relname(e1), exp_name(e1), NULL) == e1)) {
    3853         449 :                                                 sql_exp *ne = exp_alias(v->sql->sa, exp_relname(e1), exp_name(e1), exp_relname(e2), exp_name(e2), exp_subtype(e2), e2->card, has_nil(e2), is_unique(e2), is_intern(e1));
    3854             : 
    3855         449 :                                                 ne = exp_propagate(v->sql->sa, ne, e1);
    3856         449 :                                                 exp_prop_alias(v->sql->sa, ne, e1);
    3857             :                                                 e1 = ne;
    3858         449 :                                                 break;
    3859             :                                         }
    3860             :                                 }
    3861             :                         }
    3862        4951 :                         append(nexps, e1);
    3863             :                 }
    3864         425 :                 rel->exps = nexps;
    3865             :         }
    3866             :         return rel;
    3867             : }
    3868             : 
    3869             : static list *
    3870       14985 : exps_merge_select_rse( mvc *sql, list *l, list *r, bool *merged)
    3871             : {
    3872             :         node *n, *m, *o;
    3873             :         list *nexps = NULL, *lexps, *rexps;
    3874       14985 :         bool lmerged = true, rmerged = true;
    3875             : 
    3876       14985 :         lexps = new_exp_list(sql->sa);
    3877       31706 :         for (n = l->h; n; n = n->next) {
    3878       16721 :                 sql_exp *e = n->data;
    3879             : 
    3880       16721 :                 if (e->type == e_cmp && e->flag == cmp_or && !is_anti(e) && !is_semantics(e)) {
    3881        5225 :                         lmerged = false;
    3882        5225 :                         list *nexps = exps_merge_select_rse(sql, e->l, e->r, &lmerged);
    3883        5744 :                         for (o = nexps->h; o; o = o->next)
    3884         519 :                                 append(lexps, o->data);
    3885             :                 } else {
    3886       11496 :                         append(lexps, e);
    3887             :                 }
    3888             :         }
    3889       14985 :         if (lmerged)
    3890        9823 :                 lmerged = (list_length(lexps) == 1);
    3891       14985 :         rexps = new_exp_list(sql->sa);
    3892       32375 :         for (n = r->h; n; n = n->next) {
    3893       17390 :                 sql_exp *e = n->data;
    3894             : 
    3895       17390 :                 if (e->type == e_cmp && e->flag == cmp_or && !is_anti(e) && !is_semantics(e)) {
    3896        1039 :                         rmerged = false;
    3897        1039 :                         list *nexps = exps_merge_select_rse(sql, e->l, e->r, &rmerged);
    3898        1044 :                         for (o = nexps->h; o; o = o->next)
    3899           5 :                                 append(rexps, o->data);
    3900             :                 } else {
    3901       16351 :                         append(rexps, e);
    3902             :                 }
    3903             :         }
    3904       14985 :         if (rmerged)
    3905       13953 :                 rmerged = (list_length(r) == 1);
    3906             : 
    3907       14985 :         nexps = new_exp_list(sql->sa);
    3908             : 
    3909             :         /* merge merged lists first ? */
    3910       27000 :         for (n = lexps->h; n; n = n->next) {
    3911       12015 :                 sql_exp *le = n->data, *re, *fnd = NULL;
    3912             : 
    3913       12015 :                 if (le->type != e_cmp || le->flag == cmp_or || is_anti(le) || is_semantics(le) || is_symmetric(le))
    3914        1332 :                         continue;
    3915       21848 :                 for (m = rexps->h; !fnd && m; m = m->next) {
    3916       11165 :                         re = m->data;
    3917       11165 :                         if (exps_match_col_exps(le, re))
    3918             :                                 fnd = re;
    3919             :                 }
    3920       10683 :                 if (fnd && (is_anti(fnd) || is_semantics(fnd)))
    3921        1318 :                         continue;
    3922             :                 /* cases
    3923             :                  * 1) 2 values (cmp_equal)
    3924             :                  * 2) 1 value (cmp_equal), and cmp_in
    3925             :                  *      (also cmp_in, cmp_equal)
    3926             :                  * 3) 2 cmp_in
    3927             :                  * 4) ranges
    3928             :                  */
    3929        9365 :                 if (fnd) {
    3930             :                         re = fnd;
    3931             :                         fnd = NULL;
    3932        1570 :                         if (is_anti(le) || is_anti(re) || is_symmetric(re))
    3933           0 :                                 continue;
    3934        2084 :                         if (le->flag == cmp_equal && re->flag == cmp_equal) {
    3935         514 :                                 list *exps = new_exp_list(sql->sa);
    3936             : 
    3937         514 :                                 append(exps, le->r);
    3938         514 :                                 append(exps, re->r);
    3939         514 :                                 fnd = exp_in(sql->sa, le->l, exps, cmp_in);
    3940        1227 :                         } else if (le->flag == cmp_equal && re->flag == cmp_in){
    3941         171 :                                 list *exps = new_exp_list(sql->sa);
    3942             : 
    3943         171 :                                 append(exps, le->r);
    3944         171 :                                 list_merge(exps, re->r, NULL);
    3945         171 :                                 fnd = exp_in(sql->sa, le->l, exps, cmp_in);
    3946        1224 :                         } else if (le->flag == cmp_in && re->flag == cmp_equal){
    3947         339 :                                 list *exps = new_exp_list(sql->sa);
    3948             : 
    3949         339 :                                 append(exps, re->r);
    3950         339 :                                 list_merge(exps, le->r, NULL);
    3951         339 :                                 fnd = exp_in(sql->sa, le->l, exps, cmp_in);
    3952         666 :                         } else if (le->flag == cmp_in && re->flag == cmp_in){
    3953         120 :                                 list *exps = new_exp_list(sql->sa);
    3954             : 
    3955         120 :                                 list_merge(exps, le->r, NULL);
    3956         120 :                                 list_merge(exps, re->r, NULL);
    3957         120 :                                 fnd = exp_in(sql->sa, le->l, exps, cmp_in);
    3958         426 :                         } else if (le->f && re->f && /* merge ranges */
    3959          18 :                                    le->flag == re->flag && le->flag <= cmp_lt) {
    3960             :                                 sql_exp *mine = NULL, *maxe = NULL;
    3961             : 
    3962          18 :                                 if (!(mine = rel_binop_(sql, NULL, le->r, re->r, "sys", "sql_min", card_value))) {
    3963           0 :                                         sql->session->status = 0;
    3964           0 :                                         sql->errstr[0] = '\0';
    3965           0 :                                         continue;
    3966             :                                 }
    3967          18 :                                 if (!(maxe = rel_binop_(sql, NULL, le->f, re->f, "sys", "sql_max", card_value))) {
    3968           0 :                                         sql->session->status = 0;
    3969           0 :                                         sql->errstr[0] = '\0';
    3970           0 :                                         continue;
    3971             :                                 }
    3972          18 :                                 fnd = exp_compare2(sql->sa, le->l, mine, maxe, le->flag, 0);
    3973          18 :                                 lmerged = false;
    3974             :                         }
    3975        1162 :                         if (fnd) {
    3976        1162 :                                 append(nexps, fnd);
    3977        2199 :                                 *merged = (fnd && lmerged && rmerged);
    3978             :                         }
    3979             :                 }
    3980             :         }
    3981       14985 :         return nexps;
    3982             : }
    3983             : 
    3984             : static sql_exp *
    3985    18016136 : rel_merge_project_rse(visitor *v, sql_rel *rel, sql_exp *e, int depth)
    3986             : {
    3987             :         (void) depth;
    3988             : 
    3989    18016136 :         if (is_simple_project(rel->op) && is_func(e->type) && e->l) {
    3990             :                 list *fexps = e->l;
    3991      672225 :                 sql_subfunc *f = e->f;
    3992             : 
    3993             :                 /* is and function */
    3994      672225 :                 if (strcmp(f->func->base.name, "and") == 0 && list_length(fexps) == 2) {
    3995       11994 :                         sql_exp *l = list_fetch(fexps, 0);
    3996       11994 :                         sql_exp *r = list_fetch(fexps, 1);
    3997             : 
    3998             :                         /* check merge into single between */
    3999       11994 :                         if (is_func(l->type) && is_func(r->type)) {
    4000       10644 :                                 list *lfexps = l->l;
    4001       10644 :                                 list *rfexps = r->l;
    4002       10644 :                                 sql_subfunc *lff = l->f;
    4003       10644 :                                 sql_subfunc *rff = r->f;
    4004             : 
    4005       10644 :                                 if (((strcmp(lff->func->base.name, ">=") == 0 || strcmp(lff->func->base.name, ">") == 0) && list_length(lfexps) == 2) &&
    4006         526 :                                     ((strcmp(rff->func->base.name, "<=") == 0 || strcmp(rff->func->base.name, "<") == 0) && list_length(rfexps) == 2)) {
    4007         126 :                                         sql_exp *le = list_fetch(lfexps,0), *lf = list_fetch(rfexps,0);
    4008         126 :                                         int c_le = is_numeric_upcast(le), c_lf = is_numeric_upcast(lf);
    4009             : 
    4010         126 :                                         if (exp_equal(c_le?le->l:le, c_lf?lf->l:lf) == 0) {
    4011         107 :                                                 sql_exp *re = list_fetch(lfexps, 1), *rf = list_fetch(rfexps, 1), *ne = NULL;
    4012             :                                                 sql_subtype super;
    4013             : 
    4014         107 :                                                 supertype(&super, exp_subtype(le), exp_subtype(lf)); /* le/re and lf/rf must have the same type */
    4015         214 :                                                 if (!(le = exp_check_type(v->sql, &super, rel, le, type_equal)) ||
    4016         214 :                                                         !(re = exp_check_type(v->sql, &super, rel, re, type_equal)) ||
    4017         107 :                                                         !(rf = exp_check_type(v->sql, &super, rel, rf, type_equal))) {
    4018           0 :                                                                 v->sql->session->status = 0;
    4019           0 :                                                                 v->sql->errstr[0] = 0;
    4020           0 :                                                                 return e;
    4021             :                                                         }
    4022         107 :                                                 if ((ne = exp_compare2(v->sql->sa, le, re, rf, compare_funcs2range(lff->func->base.name, rff->func->base.name), 0))) {
    4023         107 :                                                         if (exp_name(e))
    4024           0 :                                                                 exp_prop_alias(v->sql->sa, ne, e);
    4025             :                                                         e = ne;
    4026         107 :                                                         v->changes++;
    4027             :                                                 }
    4028             :                                         }
    4029             :                                 }
    4030             :                         }
    4031             :                 }
    4032             :         }
    4033             :         return e;
    4034             : }
    4035             : 
    4036             : /* merge related sub expressions
    4037             :  *
    4038             :  * ie   (x = a and y > 1 and y < 5) or
    4039             :  *      (x = c and y > 1 and y < 10) or
    4040             :  *      (x = e and y > 1 and y < 20)
    4041             :  * ->
    4042             :  *     ((x = a and y > 1 and y < 5) or
    4043             :  *      (x = c and y > 1 and y < 10) or
    4044             :  *      (x = e and y > 1 and y < 20)) and
    4045             :  *       x in (a,c,e) and
    4046             :  *       y > 1 and y < 20
    4047             :  *
    4048             :  * for single expression or's we can do better
    4049             :  *              x in (a, b, c) or x in (d, e, f)
    4050             :  *              ->
    4051             :  *              x in (a, b, c, d, e, f)
    4052             :  * */
    4053             : static inline sql_rel *
    4054      577605 : rel_merge_select_rse(visitor *v, sql_rel *rel)
    4055             : {
    4056             :         /* only execute once per select */
    4057      577605 :         if ((is_select(rel->op) || is_join(rel->op) || is_semi(rel->op)) && rel->exps && !rel->used) {
    4058             :                 node *n, *o;
    4059      170805 :                 list *nexps = new_exp_list(v->sql->sa);
    4060             : 
    4061      361748 :                 for (n=rel->exps->h; n; n = n->next) {
    4062      190943 :                         sql_exp *e = n->data;
    4063             : 
    4064      199664 :                         if (e->type == e_cmp && e->flag == cmp_or && !is_anti(e) && !is_semantics(e)) {
    4065             :                                 /* possibly merge related expressions */
    4066        8721 :                                 bool merged = false;
    4067             : 
    4068        8721 :                                 list *ps = exps_merge_select_rse(v->sql, e->l, e->r, &merged);
    4069        9359 :                                 for (o = ps->h; o; o = o->next)
    4070         638 :                                         append(nexps, o->data);
    4071        8721 :                                 if (merged)
    4072          64 :                                         v->changes++;
    4073             :                                 else
    4074        8657 :                                         append(nexps, e);
    4075             :                         } else {
    4076      182222 :                                 append(nexps, e);
    4077             :                         }
    4078             :                 }
    4079      170805 :                 rel->exps = nexps;
    4080      170805 :                 rel->used = 1;
    4081             :         }
    4082      577605 :         return rel;
    4083             : }
    4084             : 
    4085             : static sql_exp *list_exps_uses_exp(list *exps, const char *rname, const char *name);
    4086             : 
    4087             : static sql_exp*
    4088      145121 : exp_uses_exp(sql_exp *e, const char *rname, const char *name)
    4089             : {
    4090             :         sql_exp *res = NULL;
    4091             : 
    4092      145156 :         switch (e->type) {
    4093             :                 case e_psm:
    4094             :                         break;
    4095          68 :                 case e_atom: {
    4096          68 :                         if (e->f)
    4097           0 :                                 return list_exps_uses_exp(e->f, rname, name);
    4098             :                 } break;
    4099          34 :                 case e_convert:
    4100          34 :                         return exp_uses_exp(e->l, rname, name);
    4101       99344 :                 case e_column: {
    4102       99344 :                         if (e->l && rname && strcmp(e->l, rname) == 0 &&
    4103       48556 :                                 e->r && name && strcmp(e->r, name) == 0)
    4104             :                                 return e;
    4105       50990 :                         if (!e->l && !rname &&
    4106         396 :                                 e->r && name && strcmp(e->r, name) == 0)
    4107         137 :                                 return e;
    4108             :                 } break;
    4109       45556 :                 case e_func:
    4110             :                 case e_aggr: {
    4111       45556 :                         if (e->l)
    4112       44669 :                                 return list_exps_uses_exp(e->l, rname, name);
    4113             :                 }       break;
    4114         154 :                 case e_cmp: {
    4115         154 :                         if (e->flag == cmp_in || e->flag == cmp_notin) {
    4116           0 :                                 if ((res = exp_uses_exp(e->l, rname, name)))
    4117             :                                         return res;
    4118           0 :                                 return list_exps_uses_exp(e->r, rname, name);
    4119         154 :                         } else if (e->flag == cmp_or || e->flag == cmp_filter) {
    4120           0 :                                 if ((res = list_exps_uses_exp(e->l, rname, name)))
    4121             :                                         return res;
    4122           0 :                                 return list_exps_uses_exp(e->r, rname, name);
    4123             :                         } else {
    4124         154 :                                 if ((res = exp_uses_exp(e->l, rname, name)))
    4125             :                                         return res;
    4126         154 :                                 if ((res = exp_uses_exp(e->r, rname, name)))
    4127             :                                         return res;
    4128         152 :                                 if (e->f)
    4129             :                                         return exp_uses_exp(e->f, rname, name);
    4130             :                         }
    4131             :                 } break;
    4132             :         }
    4133             :         return NULL;
    4134             : }
    4135             : 
    4136             : static sql_exp *
    4137       93751 : list_exps_uses_exp(list *exps, const char *rname, const char *name)
    4138             : {
    4139             :         sql_exp *res = NULL;
    4140             : 
    4141       93751 :         if (!exps)
    4142             :                 return NULL;
    4143      238559 :         for (node *n = exps->h; n && !res; n = n->next) {
    4144      144813 :                 sql_exp *e = n->data;
    4145      144813 :                 res = exp_uses_exp(e, rname, name);
    4146             :         }
    4147             :         return res;
    4148             : }
    4149             : 
    4150             : /* find in the list of expression an expression which uses e */
    4151             : static sql_exp *
    4152        1120 : exps_uses_exp(list *exps, sql_exp *e)
    4153             : {
    4154        1120 :         return list_exps_uses_exp(exps, exp_relname(e), exp_name(e));
    4155             : }
    4156             : 
    4157             : static bool
    4158       47954 : exps_uses_any(list *exps, list *l)
    4159             : {
    4160             :         bool uses_any = false;
    4161             : 
    4162       95916 :         for (node *n = l->h; n && !uses_any; n = n->next) {
    4163       47962 :                 sql_exp *e = n->data;
    4164       47962 :                 uses_any |= list_exps_uses_exp(exps, exp_relname(e), exp_name(e)) != NULL;
    4165             :         }
    4166             : 
    4167       47954 :         return uses_any;
    4168             : }
    4169             : 
    4170             : /*
    4171             :  * Rewrite aggregations over union all.
    4172             :  *      groupby ([ union all (a, b) ], [gbe], [ count, sum ] )
    4173             :  *
    4174             :  * into
    4175             :  *      groupby ( [ union all( groupby( a, [gbe], [ count, sum] ), [ groupby( b, [gbe], [ count, sum] )) , [gbe], [sum, sum] )
    4176             :  */
    4177             : static inline sql_rel *
    4178      200583 : rel_push_aggr_down(visitor *v, sql_rel *rel)
    4179             : {
    4180      200583 :         if (rel->op == op_groupby && rel->l) {
    4181             :                 sql_rel *u = rel->l, *ou = u;
    4182             :                 sql_rel *g = rel;
    4183      200583 :                 sql_rel *ul = u->l;
    4184             :                 sql_rel *ur = u->r;
    4185             :                 node *n, *m;
    4186             :                 list *lgbe = NULL, *rgbe = NULL, *gbe = NULL, *exps = NULL;
    4187             : 
    4188      200583 :                 if (u->op == op_project)
    4189             :                         u = u->l;
    4190             : 
    4191      200583 :                 if (!u || !is_union(u->op) || need_distinct(u) || is_single(u) || !u->exps || rel_is_ref(u))
    4192             :                         return rel;
    4193             : 
    4194        2989 :                 ul = u->l;
    4195        2989 :                 ur = u->r;
    4196             : 
    4197             :                 /* make sure we don't create group by on group by's */
    4198        2989 :                 if (ul->op == op_groupby || ur->op == op_groupby)
    4199             :                         return rel;
    4200             : 
    4201             :                 /* distinct should be done over the full result */
    4202        2429 :                 for (n = g->exps->h; n; n = n->next) {
    4203        1720 :                         sql_exp *e = n->data;
    4204        1720 :                         sql_subfunc *af = e->f;
    4205             : 
    4206        1720 :                         if (e->type == e_atom ||
    4207        1720 :                             e->type == e_func ||
    4208         737 :                            (e->type == e_aggr &&
    4209         737 :                            ((strcmp(af->func->base.name, "sum") &&
    4210         547 :                              strcmp(af->func->base.name, "count") &&
    4211         172 :                              strcmp(af->func->base.name, "min") &&
    4212         737 :                              strcmp(af->func->base.name, "max")) ||
    4213             :                            need_distinct(e))))
    4214             :                                 return rel;
    4215             :                 }
    4216             : 
    4217         709 :                 ul = rel_dup(ul);
    4218         709 :                 ur = rel_dup(ur);
    4219         709 :                 if (!is_project(ul->op))
    4220          18 :                         ul = rel_project(v->sql->sa, ul,
    4221             :                                 rel_projections(v->sql, ul, NULL, 1, 1));
    4222         709 :                 if (!is_project(ur->op))
    4223          21 :                         ur = rel_project(v->sql->sa, ur,
    4224             :                                 rel_projections(v->sql, ur, NULL, 1, 1));
    4225         709 :                 rel_rename_exps(v->sql, u->exps, ul->exps);
    4226         709 :                 rel_rename_exps(v->sql, u->exps, ur->exps);
    4227         709 :                 if (u != ou) {
    4228         276 :                         ul = rel_project(v->sql->sa, ul, NULL);
    4229         276 :                         ul->exps = exps_copy(v->sql, ou->exps);
    4230         276 :                         rel_rename_exps(v->sql, ou->exps, ul->exps);
    4231         276 :                         ur = rel_project(v->sql->sa, ur, NULL);
    4232         276 :                         ur->exps = exps_copy(v->sql, ou->exps);
    4233         276 :                         rel_rename_exps(v->sql, ou->exps, ur->exps);
    4234             :                 }
    4235             : 
    4236         709 :                 if (g->r && list_length(g->r) > 0) {
    4237         580 :                         list *gbe = g->r;
    4238             : 
    4239         580 :                         lgbe = exps_copy(v->sql, gbe);
    4240         580 :                         rgbe = exps_copy(v->sql, gbe);
    4241             :                 }
    4242         709 :                 ul = rel_groupby(v->sql, ul, NULL);
    4243         709 :                 ul->r = lgbe;
    4244         709 :                 ul->nrcols = g->nrcols;
    4245         709 :                 ul->card = g->card;
    4246         709 :                 ul->exps = exps_copy(v->sql, g->exps);
    4247         709 :                 ul->nrcols = list_length(ul->exps);
    4248             : 
    4249         709 :                 ur = rel_groupby(v->sql, ur, NULL);
    4250         709 :                 ur->r = rgbe;
    4251         709 :                 ur->nrcols = g->nrcols;
    4252         709 :                 ur->card = g->card;
    4253         709 :                 ur->exps = exps_copy(v->sql, g->exps);
    4254         709 :                 ur->nrcols = list_length(ur->exps);
    4255             : 
    4256             :                 /* group by on primary keys which define the partioning scheme
    4257             :                  * don't need a finalizing group by */
    4258             :                 /* how to check if a partition is based on some primary key ?
    4259             :                  * */
    4260         709 :                 if (!list_empty(rel->r)) {
    4261        1550 :                         for (node *n = ((list*)rel->r)->h; n; n = n->next) {
    4262         970 :                                 sql_exp *e = n->data;
    4263             :                                 sql_column *c = NULL;
    4264             : 
    4265         970 :                                 if ((c = exp_is_pkey(rel, e)) && partition_find_part(v->sql->session->tr, c->t, NULL)) {
    4266             :                                         /* check if key is partition key */
    4267           0 :                                         v->changes++;
    4268           0 :                                         return rel_inplace_setop(v->sql, rel, ul, ur, op_union,
    4269             :                                                                                          rel_projections(v->sql, rel, NULL, 1, 1));
    4270             :                                 }
    4271             :                         }
    4272             :                 }
    4273             : 
    4274         709 :                 if (rel->r) {
    4275             :                         list *ogbe = rel->r;
    4276             : 
    4277         580 :                         gbe = new_exp_list(v->sql->sa);
    4278        1550 :                         for (n = ogbe->h; n; n = n->next) {
    4279         970 :                                 sql_exp *e = n->data, *ne;
    4280             : 
    4281             :                                 /* group by in aggreation list */
    4282         970 :                                 ne = exps_uses_exp( rel->exps, e);
    4283         970 :                                 if (ne)
    4284         962 :                                         ne = list_find_exp( ul->exps, ne);
    4285         970 :                                 if (!ne) {
    4286             :                                         /* e only in the ul/ur->r (group by list) */
    4287           9 :                                         ne = exp_ref(v->sql, e);
    4288           9 :                                         list_append(ul->exps, ne);
    4289           9 :                                         ne = exp_ref(v->sql, e);
    4290           9 :                                         list_append(ur->exps, ne);
    4291             :                                 }
    4292         970 :                                 assert(ne);
    4293         970 :                                 ne = exp_ref(v->sql, ne);
    4294         970 :                                 append(gbe, ne);
    4295             :                         }
    4296             :                 }
    4297             : 
    4298         709 :                 u = rel_setop(v->sql->sa, ul, ur, op_union);
    4299         709 :                 rel_setop_set_exps(v->sql, u, rel_projections(v->sql, ul, NULL, 1, 1), false);
    4300         709 :                 set_processed(u);
    4301             : 
    4302         709 :                 exps = new_exp_list(v->sql->sa);
    4303        2309 :                 for (n = u->exps->h, m = rel->exps->h; n && m; n = n->next, m = m->next) {
    4304        1600 :                         sql_exp *ne, *e = n->data, *oa = m->data;
    4305             : 
    4306        1600 :                         if (oa->type == e_aggr) {
    4307         620 :                                 sql_subfunc *f = oa->f;
    4308         620 :                                 int cnt = exp_aggr_is_count(oa);
    4309         620 :                                 sql_subfunc *a = sql_bind_func(v->sql, "sys", (cnt)?"sum":f->func->base.name, exp_subtype(e), NULL, F_AGGR);
    4310             : 
    4311         620 :                                 assert(a);
    4312             :                                 /* union of aggr result may have nils
    4313             :                                  * because sum/count of empty set */
    4314         620 :                                 set_has_nil(e);
    4315         620 :                                 e = exp_ref(v->sql, e);
    4316         620 :                                 ne = exp_aggr1(v->sql->sa, e, a, need_distinct(e), 1, e->card, 1);
    4317             :                         } else {
    4318         980 :                                 ne = exp_copy(v->sql, oa);
    4319             :                         }
    4320        1600 :                         exp_setname(v->sql->sa, ne, exp_find_rel_name(oa), exp_name(oa));
    4321        1600 :                         append(exps, ne);
    4322             :                 }
    4323         709 :                 v->changes++;
    4324         709 :                 return rel_inplace_groupby( rel, u, gbe, exps);
    4325             :         }
    4326             :         return rel;
    4327             : }
    4328             : 
    4329             : /*
    4330             :  * More general
    4331             :  *      groupby(
    4332             :  *       [ outer ] join(
    4333             :  *          project(
    4334             :  *            table(A) [ c1, c2, .. ]
    4335             :  *          ) [ c1, c2, identity(c2) as I, .. ],
    4336             :  *          table(B) [ c1, c2, .. ]
    4337             :  *        ) [ A.c1 = B.c1 ]
    4338             :  *      ) [ I ] [ a1, a2, .. ]
    4339             :  *
    4340             :  * ->
    4341             :  *
    4342             :  *      [ outer ] join(
    4343             :  *        project(
    4344             :  *          table(A) [ c1, c2, .. ]
    4345             :  *        ) [ c1, c2, .. ],
    4346             :  *        groupby (
    4347             :  *          table(B) [ c1, c2, .. ]
    4348             :  *        ) [ B.c1 ] [ a1, a2, .. ]
    4349             :  *      ) [ A.c1 = B.c1 ]
    4350             :  */
    4351             : static sql_rel *
    4352      135790 : gen_push_groupby_down(mvc *sql, sql_rel *rel, int *changes)
    4353             : {
    4354      135790 :         sql_rel *j = rel->l;
    4355      135790 :         list *gbe = rel->r;
    4356             : 
    4357      135790 :         if (rel->op == op_groupby && list_length(gbe) == 1 && j->op == op_join){
    4358       49873 :                 sql_rel *jl = j->l, *jr = j->r, *cr, *cl;
    4359       49873 :                 sql_exp *gb = gbe->h->data, *e;
    4360             :                 node *n;
    4361             :                 int left = 1;
    4362             :                 list *aggrs, *aliases, *gbe;
    4363             : 
    4364       49873 :                 if (!is_identity(gb, jl) && !is_identity(gb, jr))
    4365             :                         return rel;
    4366          17 :                 if (jl->op == op_project &&
    4367          12 :                     (e = list_find_exp( jl->exps, gb)) != NULL &&
    4368           6 :                      find_prop(e->p, PROP_HASHCOL) != NULL) {
    4369             :                         left = 0;
    4370             :                         cr = jr;
    4371             :                         cl = jl;
    4372          10 :                 } else if (jr->op == op_project &&
    4373          10 :                     (e = list_find_exp( jr->exps, gb)) != NULL &&
    4374           5 :                      find_prop(e->p, PROP_HASHCOL) != NULL) {
    4375             :                         left = 1;
    4376             :                         cr = jl;
    4377             :                         cl = jr;
    4378             :                 } else {
    4379           0 :                         return rel;
    4380             :                 }
    4381             : 
    4382          11 :                 if ((left && is_base(jl->op)) || (!left && is_base(jr->op))||
    4383           5 :                     (left && is_select(jl->op)) || (!left && is_select(jr->op))
    4384           3 :                     || rel_is_join_on_pkey(j, false))
    4385           8 :                         return rel;
    4386             : 
    4387             :                 /* only add aggr (based on left/right), and repeat the group by column */
    4388           3 :                 aggrs = sa_list(sql->sa);
    4389           3 :                 aliases = sa_list(sql->sa);
    4390           9 :                 if (rel->exps) for (n = rel->exps->h; n; n = n->next) {
    4391           9 :                         sql_exp *ce = n->data;
    4392             : 
    4393           9 :                         if (exp_is_atom(ce))
    4394           0 :                                 list_append(aliases, ce);
    4395           9 :                         else if (ce->type == e_column) {
    4396           6 :                                 if (rel_has_exp(cl, ce) == 0) /* collect aliases outside groupby */
    4397           6 :                                         list_append(aliases, ce);
    4398             :                                 else
    4399           0 :                                         list_append(aggrs, ce);
    4400           3 :                         } else if (ce->type == e_aggr) {
    4401           3 :                                 list *args = ce->l;
    4402             : 
    4403             :                                 /* check args are part of left/right */
    4404           3 :                                 if (!list_empty(args) && rel_has_exps(cl, args) == 0)
    4405             :                                         return rel;
    4406           0 :                                 if (rel->op != op_join && exp_aggr_is_count(ce))
    4407           0 :                                         ce->p = prop_create(sql->sa, PROP_COUNT, ce->p);
    4408           0 :                                 list_append(aggrs, ce);
    4409             :                         }
    4410             :                 }
    4411             :                 /* TODO move any column expressions (aliases) into the project list */
    4412             : 
    4413             :                 /* find gb in left or right and should be unique */
    4414           0 :                 gbe = sa_list(sql->sa);
    4415             :                 /* push groupby to right, group on join exps */
    4416           0 :                 if (j->exps) for (n = j->exps->h; n; n = n->next) {
    4417           0 :                         sql_exp *ce = n->data, *e;
    4418             : 
    4419             :                         /* get left/right hand of e_cmp */
    4420           0 :                         assert(ce->type == e_cmp);
    4421           0 :                         if (ce->flag != cmp_equal)
    4422             :                                 return rel;
    4423           0 :                         e = rel_find_exp(cr, ce->l);
    4424           0 :                         if (!e)
    4425           0 :                                 e = rel_find_exp(cr, ce->r);
    4426           0 :                         if (!e)
    4427             :                                 return rel;
    4428           0 :                         e = exp_ref(sql, e);
    4429           0 :                         list_append(gbe, e);
    4430             :                 }
    4431           0 :                 if (!left)
    4432           0 :                         cr = j->r = rel_groupby(sql, cr, gbe);
    4433             :                 else
    4434           0 :                         cr = j->l = rel_groupby(sql, cr, gbe);
    4435           0 :                 cr->exps = list_merge(cr->exps, aggrs, (fdup)NULL);
    4436           0 :                 set_processed(cr);
    4437           0 :                 if (!is_project(cl->op))
    4438           0 :                         cl = rel_project(sql->sa, cl,
    4439             :                                 rel_projections(sql, cl, NULL, 1, 1));
    4440           0 :                 cl->exps = list_merge(cl->exps, aliases, (fdup)NULL);
    4441           0 :                 set_processed(cl);
    4442           0 :                 if (!left)
    4443           0 :                         j->l = cl;
    4444             :                 else
    4445           0 :                         j->r = cl;
    4446           0 :                 rel -> l = NULL;
    4447           0 :                 rel_destroy(rel);
    4448             : 
    4449           0 :                 if (list_empty(cr->exps) && list_empty(j->exps)) { /* remove crossproduct */
    4450             :                         sql_rel *r = cl;
    4451           0 :                         if (!left)
    4452           0 :                                 j->l = NULL;
    4453             :                         else
    4454           0 :                                 j->r = NULL;
    4455           0 :                         rel_destroy(j);
    4456             :                         j = r;
    4457             :                 }
    4458           0 :                 (*changes)++;
    4459           0 :                 return j;
    4460             :         }
    4461             :         return rel;
    4462             : }
    4463             : 
    4464             : /*
    4465             :  * Rewrite group(project(join(A,Dict)[a.i==dict.i])[...dict.n])[dict.n][ ... dict.n ]
    4466             :  * into
    4467             :  *      project(join(groupby (A)[a.i],[a.i]), Dict)[a.i==dict.i])[dict.n]
    4468             :  *
    4469             :  */
    4470             : static inline sql_rel *
    4471      200583 : rel_push_groupby_down(visitor *v, sql_rel *rel)
    4472             : {
    4473      200583 :         sql_rel *p = rel->l;
    4474      200583 :         list *gbe = rel->r;
    4475             : 
    4476      200583 :         if (rel->op == op_groupby && gbe && p && is_join(p->op))
    4477       68627 :                 return gen_push_groupby_down(v->sql, rel, &v->changes);
    4478      131956 :         if (rel->op == op_groupby && gbe && p && p->op == op_project) {
    4479       67163 :                 sql_rel *j = p->l;
    4480             :                 sql_rel *jl, *jr;
    4481             :                 node *n;
    4482             : 
    4483       67163 :                 if (!j || j->op != op_join || list_length(j->exps) != 1)
    4484       65467 :                         return gen_push_groupby_down(v->sql, rel, &v->changes);
    4485        1696 :                 jl = j->l;
    4486        1696 :                 jr = j->r;
    4487             : 
    4488             :                 /* check if jr is a dict with index and var still used */
    4489        1696 :                 if (jr->op != op_basetable || jr->l || !jr->r || list_length(jr->exps) != 2)
    4490        1696 :                         return gen_push_groupby_down(v->sql, rel, &v->changes);
    4491             : 
    4492             :                 /* check if group by is done on dict column */
    4493           0 :                 for(n = gbe->h; n; n = n->next) {
    4494           0 :                         sql_exp *ge = n->data, *pe = NULL, *e = NULL;
    4495             : 
    4496             :                         /* find group by exp in project, then in dict */
    4497           0 :                         pe = rel_find_exp(p, ge);
    4498           0 :                         if (pe) /* find project exp in right hand of join, ie dict */
    4499           0 :                                 e = rel_find_exp(jr, pe);
    4500           0 :                         if (pe && e) {  /* Rewrite: join with dict after the group by */
    4501           0 :                                 list *pexps = rel_projections(v->sql, rel, NULL, 1, 1), *npexps;
    4502             :                                 node *m;
    4503           0 :                                 sql_exp *ne = j->exps->h->data; /* join exp */
    4504           0 :                                 p->l = jl;   /* Project now only on the left side of the join */
    4505             : 
    4506           0 :                                 ne = ne->l;  /* The left side of the compare is the index of the left */
    4507             : 
    4508             :                                 /* find ge reference in new projection list */
    4509           0 :                                 npexps = sa_list(v->sql->sa);
    4510           0 :                                 for (m = pexps->h; m; m = m->next) {
    4511           0 :                                         sql_exp *a = m->data;
    4512             : 
    4513           0 :                                         if (exp_refers(ge, a)) {
    4514           0 :                                                 sql_exp *sc = jr->exps->t->data;
    4515           0 :                                                 sql_exp *e = exp_ref(v->sql, sc);
    4516           0 :                                                 if (exp_name(a))
    4517           0 :                                                         exp_prop_alias(v->sql->sa, e, a);
    4518             :                                                 a = e;
    4519             :                                         }
    4520           0 :                                         append(npexps, a);
    4521             :                                 }
    4522             : 
    4523             :                                 /* find ge in aggr list */
    4524           0 :                                 for (m = rel->exps->h; m; m = m->next) {
    4525           0 :                                         sql_exp *a = m->data;
    4526             : 
    4527           0 :                                         if (exp_match_exp(a, ge) || exp_refers(ge, a)) {
    4528           0 :                                                 a = exp_ref(v->sql, ne);
    4529           0 :                                                 if (exp_name(ne))
    4530           0 :                                                         exp_prop_alias(v->sql->sa, a, ne);
    4531           0 :                                                 m->data = a;
    4532             :                                         }
    4533             :                                 }
    4534             : 
    4535             :                                 /* change alias pe, ie project out the index  */
    4536           0 :                                 pe->l = (void*)exp_relname(ne);
    4537           0 :                                 pe->r = (void*)exp_name(ne);
    4538           0 :                                 if (exp_name(ne))
    4539           0 :                                         exp_prop_alias(v->sql->sa, pe, ne);
    4540             : 
    4541             :                                 /* change alias ge */
    4542           0 :                                 ge->l = (void*)exp_relname(pe);
    4543           0 :                                 ge->r = (void*)exp_name(pe);
    4544           0 :                                 if (exp_name(pe))
    4545           0 :                                         exp_prop_alias(v->sql->sa, ge, pe);
    4546             : 
    4547             :                                 /* zap both project and groupby name hash tables (as we changed names above) */
    4548           0 :                                 list_hash_clear(rel->exps);
    4549           0 :                                 list_hash_clear((list*)rel->r);
    4550           0 :                                 list_hash_clear(p->exps);
    4551             : 
    4552             :                                 /* add join */
    4553           0 :                                 j->l = rel;
    4554           0 :                                 rel = rel_project(v->sql->sa, j, npexps);
    4555           0 :                                 v->changes++;
    4556             :                         }
    4557             :                 }
    4558             :         }
    4559             :         return rel;
    4560             : }
    4561             : 
    4562             : /*
    4563             :  * Push select down, pushes the selects through (simple) projections. Also
    4564             :  * it cleans up the projections which become useless.
    4565             :  *
    4566             :  * WARNING - Make sure to call try_remove_empty_select macro before returning so we ensure
    4567             :  * possible generated empty selects won't never be generated
    4568             :  */
    4569             : static sql_rel *
    4570     9655712 : rel_push_select_down(visitor *v, sql_rel *rel)
    4571             : {
    4572             :         list *exps = NULL;
    4573             :         sql_rel *r = NULL;
    4574             :         node *n;
    4575             : 
    4576     9655712 :         if (rel_is_ref(rel)) {
    4577      582062 :                 if (is_select(rel->op) && rel->exps) {
    4578             :                         /* add inplace empty select */
    4579        1400 :                         sql_rel *l = rel_select(v->sql->sa, rel->l, NULL);
    4580             : 
    4581        1400 :                         l->exps = rel->exps;
    4582        1400 :                         rel->exps = NULL;
    4583        1400 :                         rel->l = l;
    4584        1400 :                         v->changes++;
    4585             :                 }
    4586      582062 :                 return rel;
    4587             :         }
    4588             : 
    4589             :         /* don't make changes for empty selects */
    4590     9073650 :         if (is_select(rel->op) && list_empty(rel->exps))
    4591           2 :                 return try_remove_empty_select(v, rel);
    4592             : 
    4593             :         /* merge 2 selects */
    4594     9073648 :         r = rel->l;
    4595     9073648 :         if (is_select(rel->op) && r && r->exps && is_select(r->op) && !(rel_is_ref(r)) && !exps_have_func(rel->exps)) {
    4596          32 :                 (void)list_merge(r->exps, rel->exps, (fdup)NULL);
    4597          32 :                 rel->l = NULL;
    4598          32 :                 rel_destroy(rel);
    4599          32 :                 v->changes++;
    4600          32 :                 return try_remove_empty_select(v, r);
    4601             :         }
    4602             :         /*
    4603             :          * Push select through semi/anti join
    4604             :          *      select (semi(A,B)) == semi(select(A), B)
    4605             :          */
    4606     9073616 :         if (is_select(rel->op) && r && is_semi(r->op) && !(rel_is_ref(r))) {
    4607         112 :                 rel->l = r->l;
    4608         112 :                 r->l = rel;
    4609         112 :                 v->changes++;
    4610             :                 /*
    4611             :                  * if A has 2 references (ie used on both sides of
    4612             :                  * the semi join), we also push the select into A.
    4613             :                  */
    4614         112 :                 if (rel_is_ref(rel->l) && rel->l == rel_find_ref(r->r)){
    4615             :                         sql_rel *lx = rel->l;
    4616             :                         sql_rel *rx = r->r;
    4617           0 :                         if (lx->ref.refcnt == 2 && !rel_is_ref(rx)) {
    4618           0 :                                 while (rx->l && !rel_is_ref(rx->l) &&
    4619           0 :                                        (is_project(rx->op) ||
    4620           0 :                                         is_select(rx->op) ||
    4621           0 :                                         is_join(rx->op)))
    4622             :                                                 rx = rx->l;
    4623             :                                 /* probably we need to introduce a project */
    4624           0 :                                 rel_destroy(rel->l);
    4625           0 :                                 lx = rel_project(v->sql->sa, rel, rel_projections(v->sql, rel, NULL, 1, 1));
    4626           0 :                                 r->l = lx;
    4627           0 :                                 rx->l = rel_dup(lx);
    4628             :                         }
    4629             :                 }
    4630         112 :                 return r;
    4631             :         }
    4632     9073504 :         exps = rel->exps;
    4633             : 
    4634             :         /* push select through join */
    4635     9073504 :         if (is_select(rel->op) && r && is_join(r->op) && !rel_is_ref(r) && !is_single(r)){
    4636       40407 :                 sql_rel *jl = r->l;
    4637       40407 :                 sql_rel *jr = r->r;
    4638             :                 int left = r->op == op_join || r->op == op_left;
    4639       40407 :                 int right = r->op == op_join || r->op == op_right;
    4640             : 
    4641       40407 :                 if (r->op == op_full)
    4642             :                         return rel;
    4643             : 
    4644             :                 /* introduce selects under the join (if needed) */
    4645       40385 :                 set_processed(jl);
    4646       40385 :                 set_processed(jr);
    4647       90509 :                 for (n = exps->h; n;) {
    4648       50124 :                         node *next = n->next;
    4649       50124 :                         sql_exp *e = n->data;
    4650             : 
    4651       50124 :                         if (left && rel_rebind_exp(v->sql, jl, e)) {
    4652       35960 :                                 if (!is_select(jl->op) || rel_is_ref(jl))
    4653       32702 :                                         r->l = jl = rel_select(v->sql->sa, jl, NULL);
    4654       35960 :                                 rel_select_add_exp(v->sql->sa, jl, e);
    4655       35960 :                                 list_remove_node(exps, NULL, n);
    4656       35960 :                                 v->changes++;
    4657       14164 :                         } else if (right && rel_rebind_exp(v->sql, jr, e)) {
    4658        8283 :                                 if (!is_select(jr->op) || rel_is_ref(jr))
    4659        4820 :                                         r->r = jr = rel_select(v->sql->sa, jr, NULL);
    4660        8283 :                                 rel_select_add_exp(v->sql->sa, jr, e);
    4661        8283 :                                 list_remove_node(exps, NULL, n);
    4662        8283 :                                 v->changes++;
    4663             :                         }
    4664             :                         n = next;
    4665             :                 }
    4666             :         }
    4667             : 
    4668             :         /* merge select and cross product ? */
    4669     9073482 :         if (is_select(rel->op) && r && r->op == op_join && !rel_is_ref(r) && !is_single(r)){
    4670       22299 :                 for (n = exps->h; n;) {
    4671        1771 :                         node *next = n->next;
    4672        1771 :                         sql_exp *e = n->data;
    4673             : 
    4674        1771 :                         if (exp_is_join(e, NULL) == 0) {
    4675         644 :                                 if (!r->exps)
    4676         329 :                                         r->exps = new_exp_list(v->sql->sa);
    4677         644 :                                 append(r->exps, e);
    4678         644 :                                 list_remove_node(exps, NULL, n);
    4679         644 :                                 v->changes++;
    4680             :                         }
    4681             :                         n = next;
    4682             :                 }
    4683             :         }
    4684             : 
    4685     9073482 :         if (is_select(rel->op) && r && r->op == op_project && !rel_is_ref(r) && !is_single(r)){
    4686       60241 :                 sql_rel *pl = r->l;
    4687             :                 /* we cannot push through window functions (for safety I disabled projects over DDL too) */
    4688       60241 :                 if (pl && pl->op != op_ddl && !exps_have_unsafe(r->exps, 1)) {
    4689             :                         /* introduce selects under the project (if needed) */
    4690       52006 :                         set_processed(pl);
    4691      114348 :                         for (n = exps->h; n;) {
    4692       62342 :                                 node *next = n->next;
    4693       62342 :                                 sql_exp *e = n->data, *ne = NULL;
    4694             : 
    4695       62342 :                                 if (e->type == e_cmp) {
    4696       62314 :                                         ne = exp_push_down_prj(v->sql, e, r, pl);
    4697             : 
    4698             :                                         /* can we move it down */
    4699       62314 :                                         if (ne && ne != e && pl->exps) {
    4700       28437 :                                                 if (!is_select(pl->op) || rel_is_ref(pl))
    4701       19761 :                                                         r->l = pl = rel_select(v->sql->sa, pl, NULL);
    4702       28437 :                                                 rel_select_add_exp(v->sql->sa, pl, ne);
    4703       28437 :                                                 list_remove_node(exps, NULL, n);
    4704       28437 :                                                 v->changes++;
    4705             :                                         }
    4706             :                                 }
    4707             :                                 n = next;
    4708             :                         }
    4709             :                 }
    4710             :         }
    4711             : 
    4712             :         /* try push select under set relation */
    4713     9073482 :         if (is_select(rel->op) && r && is_set(r->op) && !list_empty(r->exps) && !rel_is_ref(r) && !is_single(r) && !list_empty(exps)) {
    4714        4884 :                 sql_rel *u = r, *ul = u->l, *ur = u->r;
    4715             : 
    4716        4884 :                 ul = rel_dup(ul);
    4717        4884 :                 ur = rel_dup(ur);
    4718        4884 :                 if (!is_project(ul->op))
    4719           0 :                         ul = rel_project(v->sql->sa, ul,
    4720             :                                 rel_projections(v->sql, ul, NULL, 1, 1));
    4721        4884 :                 if (!is_project(ur->op))
    4722           0 :                         ur = rel_project(v->sql->sa, ur,
    4723             :                                 rel_projections(v->sql, ur, NULL, 1, 1));
    4724        4884 :                 rel_rename_exps(v->sql, u->exps, ul->exps);
    4725        4884 :                 rel_rename_exps(v->sql, u->exps, ur->exps);
    4726             : 
    4727             :                 /* introduce selects under the set */
    4728        4884 :                 ul = rel_select(v->sql->sa, ul, NULL);
    4729        4884 :                 ul->exps = exps_copy(v->sql, exps);
    4730        4884 :                 ur = rel_select(v->sql->sa, ur, NULL);
    4731        4884 :                 ur->exps = exps_copy(v->sql, exps);
    4732             : 
    4733        4884 :                 rel = rel_inplace_setop(v->sql, rel, ul, ur, u->op, rel_projections(v->sql, rel, NULL, 1, 1));
    4734        4884 :                 if (need_distinct(u))
    4735          14 :                         set_distinct(rel);
    4736        4884 :                 v->changes++;
    4737             :         }
    4738             : 
    4739     9073482 :         return try_remove_empty_select(v, rel);
    4740             : }
    4741             : 
    4742             : static inline sql_rel *
    4743     4665914 : rel_push_join_exps_down(visitor *v, sql_rel *rel)
    4744             : {
    4745             :         /* push select exps part of join expressions down */
    4746     4665914 :         if ((is_innerjoin(rel->op) || is_left(rel->op) || is_right(rel->op) || is_semi(rel->op)) && !list_empty(rel->exps)) {
    4747      786349 :                 int left = is_innerjoin(rel->op) || is_right(rel->op) || is_semi(rel->op);
    4748      786349 :                 int right = is_innerjoin(rel->op) || is_left(rel->op) || is_semi(rel->op);
    4749             : 
    4750     1721152 :                 for (node *n = rel->exps->h; n;) {
    4751      934803 :                         node *next = n->next;
    4752      934803 :                         sql_exp *e = n->data;
    4753             : 
    4754      934803 :                         if (left && rel_rebind_exp(v->sql, rel->l, e)) { /* select expressions on left */
    4755        1411 :                                 sql_rel *l = rel->l;
    4756        1411 :                                 if (!is_select(l->op) || rel_is_ref(l)) {
    4757        1398 :                                         set_processed(l);
    4758        1398 :                                         rel->l = l = rel_select(v->sql->sa, rel->l, NULL);
    4759             :                                 }
    4760        1411 :                                 rel_select_add_exp(v->sql->sa, rel->l, e);
    4761        1411 :                                 list_remove_node(rel->exps, NULL, n);
    4762        1411 :                                 v->changes++;
    4763     1857657 :                         } else if (right && (rel->op != op_anti || (e->flag != mark_notin && e->flag != mark_in)) &&
    4764      924265 :                                            rel_rebind_exp(v->sql, rel->r, e)) { /* select expressions on right */
    4765         125 :                                 sql_rel *r = rel->r;
    4766         125 :                                 if (!is_select(r->op) || rel_is_ref(r)) {
    4767         120 :                                         set_processed(r);
    4768         120 :                                         rel->r = r = rel_select(v->sql->sa, rel->r, NULL);
    4769             :                                 }
    4770         125 :                                 rel_select_add_exp(v->sql->sa, rel->r, e);
    4771         125 :                                 list_remove_node(rel->exps, NULL, n);
    4772         125 :                                 v->changes++;
    4773             :                         }
    4774             :                         n = next;
    4775             :                 }
    4776      786349 :                 if (is_join(rel->op) && list_empty(rel->exps))
    4777        1171 :                         rel->exps = NULL; /* crossproduct */
    4778             :         }
    4779     4665914 :         return rel;
    4780             : }
    4781             : 
    4782             : static bool
    4783      154333 : point_select_on_unique_column(sql_rel *rel)
    4784             : {
    4785      154333 :         if (is_select(rel->op) && !list_empty(rel->exps)) {
    4786      253987 :                 for (node *n = rel->exps->h; n ; n = n->next) {
    4787      148544 :                         sql_exp *e = n->data, *el = e->l, *er = e->r, *found = NULL;
    4788             : 
    4789      148544 :                         if (is_compare(e->type) && e->flag == cmp_equal) {
    4790       60922 :                                 if (is_numeric_upcast(el))
    4791           0 :                                         el = el->l;
    4792       60922 :                                 if (is_alias(el->type) && exp_is_atom(er) && (found = rel_find_exp(rel->l, el)) &&
    4793           8 :                                         is_unique(found) && (!is_semantics(e) || !has_nil(el) || !has_nil(er)))
    4794             :                                         return true;
    4795             :                         }
    4796             :                 }
    4797             :         }
    4798             :         return false;
    4799             : }
    4800             : 
    4801             : /*
    4802             :  * A point select on an unique column reduces the number of rows to 1. If the same select is under a
    4803             :  * join, the opposite side's select can be pushed above the join.
    4804             :  */
    4805             : static sql_rel *
    4806     1905751 : rel_push_select_up(visitor *v, sql_rel *rel)
    4807             : {
    4808     1905751 :         if ((is_join(rel->op) || is_semi(rel->op)) && !rel_is_ref(rel) && !is_single(rel)) {
    4809      388720 :                 sql_rel *l = rel->l, *r = rel->r;
    4810      388720 :                 bool can_pushup_left = is_select(l->op) && !rel_is_ref(l) && !is_single(l),
    4811      388720 :                          can_pushup_right = is_select(r->op) && !rel_is_ref(r) && !is_single(r) && !is_semi(rel->op);
    4812             : 
    4813      388720 :                 if (can_pushup_left || can_pushup_right) {
    4814      101610 :                         if (can_pushup_left)
    4815       85197 :                                 can_pushup_left = point_select_on_unique_column(r);
    4816      101610 :                         if (can_pushup_right)
    4817       69136 :                                 can_pushup_right = point_select_on_unique_column(l);
    4818             : 
    4819             :                         /* if both selects retrieve one row each, it's not worth it to push both up */
    4820      101610 :                         if (can_pushup_left && !can_pushup_right) {
    4821           1 :                                 sql_rel *ll = l->l;
    4822           1 :                                 rel->l = ll;
    4823           1 :                                 l->l = rel;
    4824             :                                 rel = l;
    4825           1 :                                 assert(is_select(rel->op));
    4826           1 :                                 v->changes++;
    4827      101609 :                         } else if (!can_pushup_left && can_pushup_right) {
    4828           4 :                                 sql_rel *rl = r->l;
    4829           4 :                                 rel->r = rl;
    4830           4 :                                 r->l = rel;
    4831             :                                 rel = r;
    4832           4 :                                 assert(is_select(rel->op));
    4833           4 :                                 v->changes++;
    4834             :                         }
    4835             :                 }
    4836             :         }
    4837     1905751 :         return rel;
    4838             : }
    4839             : 
    4840             : /*
    4841             :  * Push {semi}joins down, pushes the joins through group by expressions.
    4842             :  * When the join is on the group by columns, we can push the joins left
    4843             :  * under the group by. This should only be done, iff the new semijoin would
    4844             :  * reduce the input table to the groupby. So there should be a reduction
    4845             :  * (selection) on the table A and this should be propagated to the groupby via
    4846             :  * for example a primary key.
    4847             :  *
    4848             :  * {semi}join( A, groupby( B ) [gbe][aggrs] ) [ gbe == A.x ]
    4849             :  * ->
    4850             :  * {semi}join( A, groupby( semijoin(B,A) [gbe == A.x] ) [gbe][aggrs] ) [ gbe == A.x ]
    4851             :  */
    4852             : static inline sql_rel *
    4853     2112044 : rel_push_join_down(visitor *v, sql_rel *rel)
    4854             : {
    4855     2112044 :         if (!rel_is_ref(rel) && ((is_left(rel->op) || rel->op == op_join || is_semi(rel->op)) && rel->l && rel->exps)) {
    4856      305002 :                 sql_rel *gb = rel->r, *ogb = gb, *l = NULL, *rell = rel->l;
    4857             : 
    4858      305002 :                 if (is_simple_project(gb->op) && !rel_is_ref(gb))
    4859       74411 :                         gb = gb->l;
    4860             : 
    4861      305002 :                 if (rel_is_ref(rell) || !gb || rel_is_ref(gb))
    4862             :                         return rel;
    4863             : 
    4864      290092 :                 if (is_groupby(gb->op) && gb->r && list_length(gb->r)) {
    4865          84 :                         list *exps = rel->exps, *jes = new_exp_list(v->sql->sa), *gbes = gb->r;
    4866             :                         node *n, *m;
    4867             :                         /* find out if all group by expressions are used in the join */
    4868          88 :                         for(n = gbes->h; n; n = n->next) {
    4869          85 :                                 sql_exp *gbe = n->data;
    4870             :                                 int fnd = 0;
    4871             :                                 const char *rname = NULL, *name = NULL;
    4872             : 
    4873             :                                 /* project in between, ie find alias */
    4874             :                                 /* first find expression in expression list */
    4875          85 :                                 gbe = exps_uses_exp( gb->exps, gbe);
    4876          85 :                                 if (!gbe)
    4877           2 :                                         continue;
    4878          83 :                                 if (ogb != gb)
    4879          65 :                                         gbe = exps_uses_exp( ogb->exps, gbe);
    4880          83 :                                 if (gbe) {
    4881          81 :                                         rname = exp_find_rel_name(gbe);
    4882          81 :                                         name = exp_name(gbe);
    4883             :                                 }
    4884             : 
    4885          81 :                                 if (!name)
    4886           2 :                                         return rel;
    4887             : 
    4888         201 :                                 for (m = exps->h; m && !fnd; m = m->next) {
    4889         120 :                                         sql_exp *je = m->data;
    4890             : 
    4891         120 :                                         if (je->card >= CARD_ATOM && je->type == e_cmp &&
    4892         120 :                                             !is_complex_exp(je->flag)) {
    4893             :                                                 /* expect right expression to match */
    4894         120 :                                                 sql_exp *r = je->r;
    4895             : 
    4896         120 :                                                 if (r == 0 || r->type != e_column)
    4897          10 :                                                         continue;
    4898         110 :                                                 if (r->l && rname && strcmp(r->l, rname) == 0 && strcmp(r->r, name)==0) {
    4899             :                                                         fnd = 1;
    4900          40 :                                                 } else if (!r->l && !rname  && strcmp(r->r, name)==0) {
    4901             :                                                         fnd = 1;
    4902             :                                                 }
    4903             :                                                 if (fnd) {
    4904          70 :                                                         sql_exp *le = je->l;
    4905          70 :                                                         sql_exp *re = exp_push_down_prj(v->sql, r, gb, gb->l);
    4906          70 :                                                         if (!re || (list_length(jes) == 0 && !find_prop(le->p, PROP_HASHCOL))) {
    4907             :                                                                 fnd = 0;
    4908             :                                                         } else {
    4909           2 :                                                                 int anti = is_anti(je), semantics = is_semantics(je);
    4910             : 
    4911           2 :                                                                 je = exp_compare(v->sql->sa, le, re, je->flag);
    4912           2 :                                                                 if (anti) set_anti(je);
    4913           2 :                                                                 if (semantics) set_semantics(je);
    4914           2 :                                                                 list_append(jes, je);
    4915             :                                                         }
    4916             :                                                 }
    4917             :                                         }
    4918             :                                 }
    4919          81 :                                 if (!fnd)
    4920             :                                         return rel;
    4921             :                         }
    4922           3 :                         l = rel_dup(rel->l);
    4923             : 
    4924             :                         /* push join's left side (as semijoin) down group by */
    4925           3 :                         l = gb->l = rel_crossproduct(v->sql->sa, gb->l, l, op_semi);
    4926           3 :                         l->exps = jes;
    4927           3 :                         v->changes++;
    4928           3 :                         return rel;
    4929             :                 }
    4930             :         }
    4931             :         return rel;
    4932             : }
    4933             : 
    4934             : /*
    4935             :  * Push semijoins down, pushes the semijoin through a join.
    4936             :  *
    4937             :  * semijoin( join(A, B) [ A.x == B.y ], C ) [ A.z == C.c ]
    4938             :  * ->
    4939             :  * join( semijoin(A, C) [ A.z == C.c ], B ) [ A.x == B.y ]
    4940             :  *
    4941             :  * also push simple expressions of a semijoin down if they only
    4942             :  * involve the left sided of the semijoin.
    4943             :  *
    4944             :  * in some cases the other way is usefull, ie push join down
    4945             :  * semijoin. When the join reduces (ie when there are selects on it).
    4946             :  *
    4947             :  * At the moment, we only flag changes by this optimizer on the first level of optimization
    4948             :  */
    4949             : static inline sql_rel *
    4950       46258 : rel_push_semijoin_down_or_up(visitor *v, sql_rel *rel)
    4951             : {
    4952       46258 :         int level = *(int*)v->data;
    4953             : 
    4954       46258 :         if (rel->op == op_join && rel->exps && rel->l) {
    4955       28126 :                 sql_rel *l = rel->l, *r = rel->r;
    4956             : 
    4957       28126 :                 if (is_semi(l->op) && !rel_is_ref(l) && is_select(r->op) && !rel_is_ref(r)) {
    4958          54 :                         rel->l = l->l;
    4959          54 :                         l->l = rel;
    4960          54 :                         if (level <= 0)
    4961          12 :                                 v->changes++;
    4962          54 :                         return l;
    4963             :                 }
    4964             :         }
    4965             :         /* also case with 2 joins */
    4966             :         /* join ( join ( semijoin(), table), select (table)); */
    4967       46204 :         if (rel->op == op_join && rel->exps && rel->l) {
    4968       28072 :                 sql_rel *l = rel->l, *r = rel->r;
    4969             :                 sql_rel *ll;
    4970             : 
    4971       28072 :                 if (is_join(l->op) && !rel_is_ref(l) && is_select(r->op) && !rel_is_ref(r)) {
    4972         133 :                         ll = l->l;
    4973         133 :                         if (is_semi(ll->op) && !rel_is_ref(ll)) {
    4974           1 :                                 l->l = ll->l;
    4975           1 :                                 ll->l = rel;
    4976           1 :                                 if (level <= 0)
    4977           0 :                                         v->changes++;
    4978           1 :                                 return ll;
    4979             :                         }
    4980             :                 }
    4981             :         }
    4982             :         /* first push down the expressions involving only A */
    4983       46203 :         if (rel->op == op_semi && rel->exps && rel->l) {
    4984       16030 :                 for (node *n = rel->exps->h; n;) {
    4985        9023 :                         node *next = n->next;
    4986        9023 :                         sql_exp *e = n->data;
    4987             : 
    4988        9023 :                         if (n != rel->exps->h && e->type == e_cmp && rel_rebind_exp(v->sql, rel->l, e)) {
    4989           0 :                                 sql_rel *l = rel->l;
    4990           0 :                                 if (!is_select(l->op) || rel_is_ref(l)) {
    4991           0 :                                         set_processed(l);
    4992           0 :                                         rel->l = l = rel_select(v->sql->sa, rel->l, NULL);
    4993             :                                 }
    4994           0 :                                 rel_select_add_exp(v->sql->sa, rel->l, e);
    4995           0 :                                 list_remove_node(rel->exps, NULL, n);
    4996           0 :                                 if (level <= 0)
    4997           0 :                                         v->changes++;
    4998             :                         }
    4999             :                         n = next;
    5000             :                 }
    5001             :         }
    5002       46203 :         if (rel->op == op_semi && rel->exps && rel->l) {
    5003             :                 operator_type op = rel->op, lop;
    5004             :                 node *n;
    5005             :                 sql_rel *l = rel->l, *ll = NULL, *lr = NULL;
    5006        7007 :                 sql_rel *r = rel->r;
    5007             :                 list *exps = rel->exps, *nsexps, *njexps;
    5008             :                 int left = 1, right = 1;
    5009             : 
    5010             :                 /* handle project
    5011             :                 if (l->op == op_project && !need_distinct(l))
    5012             :                         l = l->l;
    5013             :                 */
    5014             : 
    5015        7007 :                 if (!is_join(l->op) || is_full(l->op) || rel_is_ref(l) || is_single(l))
    5016             :                         return rel;
    5017             : 
    5018             :                 lop = l->op;
    5019         780 :                 ll = l->l;
    5020         780 :                 lr = l->r;
    5021             : 
    5022             :                 /* check which side is used and other exps are atoms or from right of semijoin */
    5023        1574 :                 for(n = exps->h; n; n = n->next) {
    5024         802 :                         sql_exp *sje = n->data;
    5025             : 
    5026         802 :                         if (sje->type != e_cmp || is_complex_exp(sje->flag))
    5027             :                                 return rel;
    5028             :                         /* sje->l from ll and sje->r/f from semijoin r ||
    5029             :                          * sje->l from semijoin r and sje->r/f from ll ||
    5030             :                          * sje->l from lr and sje->r/f from semijoin r ||
    5031             :                          * sje->l from semijoin r and sje->r/f from lr */
    5032        1590 :                         if (left &&
    5033        1576 :                            ((rel_rebind_exp(v->sql, ll, sje->l) && rel_rebind_exp(v->sql, rel->r, sje->r) && (!sje->f || rel_rebind_exp(v->sql, rel->r, sje->f))) ||
    5034         646 :                             (rel_rebind_exp(v->sql, rel->r, sje->l) && rel_rebind_exp(v->sql, ll, sje->r) && (!sje->f || rel_rebind_exp(v->sql, ll, sje->f)))))
    5035             :                                 right = 0;
    5036             :                         else
    5037             :                                 left = 0;
    5038        1288 :                         if (right &&
    5039        1280 :                            ((rel_rebind_exp(v->sql, lr, sje->l) && rel_rebind_exp(v->sql, rel->r, sje->r) && (!sje->f || rel_rebind_exp(v->sql, rel->r, sje->f))) ||
    5040          28 :                             (rel_rebind_exp(v->sql, rel->r, sje->l) && rel_rebind_exp(v->sql, lr, sje->r) && (!sje->f || rel_rebind_exp(v->sql, lr, sje->f)))))
    5041             :                                 left = 0;
    5042             :                         else
    5043             :                                 right = 0;
    5044         802 :                         if (!right && !left)
    5045             :                                 return rel;
    5046             :                 }
    5047         772 :                 if (left && is_right(lop))
    5048             :                         return rel;
    5049         772 :                 if (right && is_left(lop))
    5050             :                         return rel;
    5051         769 :                 nsexps = exps_copy(v->sql, rel->exps);
    5052         769 :                 njexps = exps_copy(v->sql, l->exps);
    5053         769 :                 if (left)
    5054         146 :                         l = rel_crossproduct(v->sql->sa, rel_dup(ll), rel_dup(r), op);
    5055             :                 else
    5056         623 :                         l = rel_crossproduct(v->sql->sa, rel_dup(lr), rel_dup(r), op);
    5057         769 :                 l->exps = nsexps;
    5058         769 :                 if (left)
    5059         146 :                         l = rel_crossproduct(v->sql->sa, l, rel_dup(lr), lop);
    5060             :                 else
    5061         623 :                         l = rel_crossproduct(v->sql->sa, rel_dup(ll), l, lop);
    5062         769 :                 l->exps = njexps;
    5063         769 :                 rel_destroy(rel);
    5064             :                 rel = l;
    5065         769 :                 if (level <= 0)
    5066         631 :                         v->changes++;
    5067             :         }
    5068             :         return rel;
    5069             : }
    5070             : 
    5071             : static int
    5072           0 : rel_part_nr( sql_rel *rel, sql_exp *e )
    5073             : {
    5074             :         sql_column *c = NULL;
    5075           0 :         sql_rel *bt = NULL;
    5076           0 :         assert(e->type == e_cmp);
    5077             : 
    5078           0 :         c = exp_find_column_(rel, e->l, -1, &bt);
    5079           0 :         if (!c)
    5080           0 :                 c = exp_find_column_(rel, e->r, -1, &bt);
    5081           0 :         if (!c && e->f)
    5082           0 :                 c = exp_find_column_(rel, e->f, -1, &bt);
    5083           0 :         if (!c || !bt || !rel_base_get_mergetable(bt))
    5084           0 :                 return -1;
    5085           0 :         sql_table *pp = c->t;
    5086           0 :         sql_table *mt = rel_base_get_mergetable(bt);
    5087           0 :         return find_member_pos(mt->members, pp);
    5088             : }
    5089             : 
    5090             : static int
    5091           0 : rel_uses_part_nr( sql_rel *rel, sql_exp *e, int pnr )
    5092             : {
    5093             :         sql_column *c = NULL;
    5094           0 :         sql_rel *bt = NULL;
    5095           0 :         assert(e->type == e_cmp);
    5096             : 
    5097             :         /*
    5098             :          * following case fails.
    5099             :          *
    5100             :          * semijoin( A1, union [A1, A2] )
    5101             :          * The union will never return proper column (from A2).
    5102             :          * ie need different solution (probaly pass pnr).
    5103             :          */
    5104           0 :         c = exp_find_column_(rel, e->l, pnr, &bt);
    5105           0 :         if (!c)
    5106           0 :                 c = exp_find_column_(rel, e->r, pnr, &bt);
    5107           0 :         if (c && bt && rel_base_get_mergetable(bt)) {
    5108           0 :                 sql_table *pp = c->t;
    5109           0 :                 sql_table *mt = rel_base_get_mergetable(bt);
    5110           0 :                 if (find_member_pos(mt->members, pp) == pnr)
    5111             :                         return 1;
    5112             :         }
    5113             :         /* for projects we may need to do a rename! */
    5114           0 :         if (is_project(rel->op) || is_topn(rel->op) || is_sample(rel->op))
    5115           0 :                 return rel_uses_part_nr( rel->l, e, pnr);
    5116             : 
    5117           0 :         if (is_union(rel->op) || is_join(rel->op) || is_semi(rel->op)) {
    5118           0 :                 if (rel_uses_part_nr( rel->l, e, pnr))
    5119             :                         return 1;
    5120           0 :                 if (!is_semi(rel->op) && rel_uses_part_nr( rel->r, e, pnr))
    5121           0 :                         return 1;
    5122             :         }
    5123             :         return 0;
    5124             : }
    5125             : 
    5126             : /*
    5127             :  * Push (semi)joins down unions, this is basically for merge tables, where
    5128             :  * we know that the fk-indices are split over two clustered merge tables.
    5129             :  */
    5130             : static inline sql_rel *
    5131     3866011 : rel_push_join_down_union(visitor *v, sql_rel *rel)
    5132             : {
    5133     3866011 :         if ((is_join(rel->op) && !is_outerjoin(rel->op) && !is_single(rel)) || is_semi(rel->op)) {
    5134      555621 :                 sql_rel *l = rel->l, *r = rel->r, *ol = l, *or = r;
    5135      555621 :                 list *exps = rel->exps;
    5136             :                 sql_exp *je = NULL;
    5137             : 
    5138      555621 :                 if (!l || !r || need_distinct(l) || need_distinct(r) || rel_is_ref(l) || rel_is_ref(r))
    5139             :                         return rel;
    5140      468421 :                 if (l->op == op_project)
    5141        2279 :                         l = l->l;
    5142      468421 :                 if (r->op == op_project)
    5143       59570 :                         r = r->l;
    5144             : 
    5145             :                 /* both sides only if we have a join index */
    5146      468630 :                 if (!l || !r ||(is_union(l->op) && is_union(r->op) &&
    5147         209 :                         !(je = rel_is_join_on_pkey(rel, true)))) /* aligned PKEY-FKEY JOIN */
    5148         306 :                         return rel;
    5149      468115 :                 if (is_semi(rel->op) && is_union(l->op) && !je)
    5150             :                         return rel;
    5151             : 
    5152      467366 :                 if ((is_union(l->op) && !need_distinct(l) && !is_single(l)) && !is_union(r->op)){
    5153             :                         sql_rel *nl, *nr;
    5154        4372 :                         sql_rel *ll = rel_dup(l->l), *lr = rel_dup(l->r);
    5155             : 
    5156             :                         /* join(union(a,b), c) -> union(join(a,c), join(b,c)) */
    5157        4372 :                         if (!is_project(ll->op))
    5158           2 :                                 ll = rel_project(v->sql->sa, ll,
    5159             :                                         rel_projections(v->sql, ll, NULL, 1, 1));
    5160        4372 :                         if (!is_project(lr->op))
    5161           4 :                                 lr = rel_project(v->sql->sa, lr,
    5162             :                                         rel_projections(v->sql, lr, NULL, 1, 1));
    5163        4372 :                         rel_rename_exps(v->sql, l->exps, ll->exps);
    5164        4372 :                         rel_rename_exps(v->sql, l->exps, lr->exps);
    5165        4372 :                         if (l != ol) {
    5166           1 :                                 ll = rel_project(v->sql->sa, ll, NULL);
    5167           1 :                                 ll->exps = exps_copy(v->sql, ol->exps);
    5168           1 :                                 lr = rel_project(v->sql->sa, lr, NULL);
    5169           1 :                                 lr->exps = exps_copy(v->sql, ol->exps);
    5170             :                         }
    5171        4372 :                         nl = rel_crossproduct(v->sql->sa, ll, rel_dup(or), rel->op);
    5172        4372 :                         nr = rel_crossproduct(v->sql->sa, lr, rel_dup(or), rel->op);
    5173        4372 :                         nl->exps = exps_copy(v->sql, exps);
    5174        4372 :                         nr->exps = exps_copy(v->sql, exps);
    5175        4372 :                         nl = rel_project(v->sql->sa, nl, rel_projections(v->sql, nl, NULL, 1, 1));
    5176        4372 :                         nr = rel_project(v->sql->sa, nr, rel_projections(v->sql, nr, NULL, 1, 1));
    5177        4372 :                         v->changes++;
    5178        4372 :                         return rel_inplace_setop(v->sql, rel, nl, nr, op_union, rel_projections(v->sql, rel, NULL, 1, 1));
    5179      462994 :                 } else if (is_union(l->op) && !need_distinct(l) && !is_single(l) &&
    5180           0 :                            is_union(r->op) && !need_distinct(r) && !is_single(r) && je) {
    5181             :                         sql_rel *nl, *nr;
    5182           0 :                         sql_rel *ll = rel_dup(l->l), *lr = rel_dup(l->r);
    5183           0 :                         sql_rel *rl = rel_dup(r->l), *rr = rel_dup(r->r);
    5184             : 
    5185             :                         /* join(union(a,b), union(c,d)) -> union(join(a,c), join(b,d)) */
    5186           0 :                         if (!is_project(ll->op))
    5187           0 :                                 ll = rel_project(v->sql->sa, ll,
    5188             :                                         rel_projections(v->sql, ll, NULL, 1, 1));
    5189           0 :                         if (!is_project(lr->op))
    5190           0 :                                 lr = rel_project(v->sql->sa, lr,
    5191             :                                         rel_projections(v->sql, lr, NULL, 1, 1));
    5192           0 :                         rel_rename_exps(v->sql, l->exps, ll->exps);
    5193           0 :                         rel_rename_exps(v->sql, l->exps, lr->exps);
    5194           0 :                         if (l != ol) {
    5195           0 :                                 ll = rel_project(v->sql->sa, ll, NULL);
    5196           0 :                                 ll->exps = exps_copy(v->sql, ol->exps);
    5197           0 :                                 lr = rel_project(v->sql->sa, lr, NULL);
    5198           0 :                                 lr->exps = exps_copy(v->sql, ol->exps);
    5199             :                         }
    5200           0 :                         if (!is_project(rl->op))
    5201           0 :                                 rl = rel_project(v->sql->sa, rl,
    5202             :                                         rel_projections(v->sql, rl, NULL, 1, 1));
    5203           0 :                         if (!is_project(rr->op))
    5204           0 :                                 rr = rel_project(v->sql->sa, rr,
    5205             :                                         rel_projections(v->sql, rr, NULL, 1, 1));
    5206           0 :                         rel_rename_exps(v->sql, r->exps, rl->exps);
    5207           0 :                         rel_rename_exps(v->sql, r->exps, rr->exps);
    5208           0 :                         if (r != or) {
    5209           0 :                                 rl = rel_project(v->sql->sa, rl, NULL);
    5210           0 :                                 rl->exps = exps_copy(v->sql, or->exps);
    5211           0 :                                 rr = rel_project(v->sql->sa, rr, NULL);
    5212           0 :                                 rr->exps = exps_copy(v->sql, or->exps);
    5213             :                         }
    5214           0 :                         nl = rel_crossproduct(v->sql->sa, ll, rl, rel->op);
    5215           0 :                         nr = rel_crossproduct(v->sql->sa, lr, rr, rel->op);
    5216           0 :                         nl->exps = exps_copy(v->sql, exps);
    5217           0 :                         nr->exps = exps_copy(v->sql, exps);
    5218           0 :                         nl = rel_project(v->sql->sa, nl, rel_projections(v->sql, nl, NULL, 1, 1));
    5219           0 :                         nr = rel_project(v->sql->sa, nr, rel_projections(v->sql, nr, NULL, 1, 1));
    5220           0 :                         v->changes++;
    5221           0 :                         return rel_inplace_setop(v->sql, rel, nl, nr, op_union, rel_projections(v->sql, rel, NULL, 1, 1));
    5222      462994 :                 } else if (!is_union(l->op) &&
    5223      462986 :                            is_union(r->op) && !need_distinct(r) && !is_single(r) &&
    5224             :                            !is_semi(rel->op)) {
    5225             :                         sql_rel *nl, *nr;
    5226        1268 :                         sql_rel *rl = rel_dup(r->l), *rr = rel_dup(r->r);
    5227             : 
    5228             :                         /* join(a, union(b,c)) -> union(join(a,b), join(a,c)) */
    5229        1268 :                         if (!is_project(rl->op))
    5230           7 :                                 rl = rel_project(v->sql->sa, rl,
    5231             :                                         rel_projections(v->sql, rl, NULL, 1, 1));
    5232        1268 :                         if (!is_project(rr->op))
    5233          18 :                                 rr = rel_project(v->sql->sa, rr,
    5234             :                                         rel_projections(v->sql, rr, NULL, 1, 1));
    5235        1268 :                         rel_rename_exps(v->sql, r->exps, rl->exps);
    5236        1268 :                         rel_rename_exps(v->sql, r->exps, rr->exps);
    5237        1268 :                         if (r != or) {
    5238          82 :                                 rl = rel_project(v->sql->sa, rl, NULL);
    5239          82 :                                 rl->exps = exps_copy(v->sql, or->exps);
    5240          82 :                                 rr = rel_project(v->sql->sa, rr, NULL);
    5241          82 :                                 rr->exps = exps_copy(v->sql, or->exps);
    5242             :                         }
    5243        1268 :                         nl = rel_crossproduct(v->sql->sa, rel_dup(ol), rl, rel->op);
    5244        1268 :                         nr = rel_crossproduct(v->sql->sa, rel_dup(ol), rr, rel->op);
    5245        1268 :                         nl->exps = exps_copy(v->sql, exps);
    5246        1268 :                         nr->exps = exps_copy(v->sql, exps);
    5247        1268 :                         nl = rel_project(v->sql->sa, nl, rel_projections(v->sql, nl, NULL, 1, 1));
    5248        1268 :                         nr = rel_project(v->sql->sa, nr, rel_projections(v->sql, nr, NULL, 1, 1));
    5249        1268 :                         v->changes++;
    5250        1268 :                         return rel_inplace_setop(v->sql, rel, nl, nr, op_union, rel_projections(v->sql, rel, NULL, 1, 1));
    5251             :                 /* {semi}join ( A1, union (A2, B)) [A1.partkey = A2.partkey] ->
    5252             :                  * {semi}join ( A1, A2 )
    5253             :                  * and
    5254             :                  * {semi}join ( A1, union (B, A2)) [A1.partkey = A2.partkey] ->
    5255             :                  * {semi}join ( A1, A2 )
    5256             :                  * (ie a single part on the left)
    5257             :                  *
    5258             :                  * Howto detect that a relation isn't matching.
    5259             :                  *
    5260             :                  * partitioning is currently done only on pkey/fkey's
    5261             :                  * ie only matching per part if join is on pkey/fkey (parts)
    5262             :                  *
    5263             :                  * and part numbers should match.
    5264             :                  *
    5265             :                  * */
    5266      461726 :                 } else if (!is_union(l->op) &&
    5267      461718 :                            is_union(r->op) && !need_distinct(r) && !is_single(r) &&
    5268         932 :                            is_semi(rel->op) && (je = rel_is_join_on_pkey(rel, true))) {
    5269             :                         /* use first join expression, to find part nr */
    5270           0 :                         int lpnr = rel_part_nr(l, je);
    5271           0 :                         sql_rel *rl = r->l;
    5272           0 :                         sql_rel *rr = r->r;
    5273             : 
    5274           0 :                         if (lpnr < 0)
    5275             :                                 return rel;
    5276             :                         /* case 1: uses left not right */
    5277           0 :                         if (rel_uses_part_nr(rl, je, lpnr) &&
    5278           0 :                            !rel_uses_part_nr(rr, je, lpnr)) {
    5279             :                                 sql_rel *nl;
    5280             : 
    5281           0 :                                 rl = rel_dup(rl);
    5282           0 :                                 if (!is_project(rl->op))
    5283           0 :                                         rl = rel_project(v->sql->sa, rl,
    5284             :                                         rel_projections(v->sql, rl, NULL, 1, 1));
    5285           0 :                                 rel_rename_exps(v->sql, r->exps, rl->exps);
    5286           0 :                                 if (r != or) {
    5287           0 :                                         rl = rel_project(v->sql->sa, rl, NULL);
    5288           0 :                                         rl->exps = exps_copy(v->sql, or->exps);
    5289             :                                 }
    5290           0 :                                 nl = rel_crossproduct(v->sql->sa, rel_dup(ol), rl, rel->op);
    5291           0 :                                 nl->exps = exps_copy(v->sql, exps);
    5292           0 :                                 v->changes++;
    5293           0 :                                 return rel_inplace_project(v->sql->sa, rel, nl, rel_projections(v->sql, rel, NULL, 1, 1));
    5294             :                         /* case 2: uses right not left */
    5295           0 :                         } else if (!rel_uses_part_nr(rl, je, lpnr) &&
    5296           0 :                                     rel_uses_part_nr(rr, je, lpnr)) {
    5297             :                                 sql_rel *nl;
    5298             : 
    5299           0 :                                 rr = rel_dup(rr);
    5300           0 :                                 if (!is_project(rr->op))
    5301           0 :                                         rr = rel_project(v->sql->sa, rr,
    5302             :                                                 rel_projections(v->sql, rr, NULL, 1, 1));
    5303           0 :                                 rel_rename_exps(v->sql, r->exps, rr->exps);
    5304           0 :                                 if (r != or) {
    5305           0 :                                         rr = rel_project(v->sql->sa, rr, NULL);
    5306           0 :                                         rr->exps = exps_copy(v->sql, or->exps);
    5307             :                                 }
    5308           0 :                                 nl = rel_crossproduct(v->sql->sa, rel_dup(ol), rr, rel->op);
    5309           0 :                                 nl->exps = exps_copy(v->sql, exps);
    5310           0 :                                 v->changes++;
    5311           0 :                                 return rel_inplace_project(v->sql->sa, rel, nl, rel_projections(v->sql, rel, NULL, 1, 1));
    5312             :                         }
    5313             :                 }
    5314             :         }
    5315             :         return rel;
    5316             : }
    5317             : 
    5318             : static inline sql_rel *
    5319     4665914 : rel_push_join_down_outer(visitor *v, sql_rel *rel)
    5320             : {
    5321     4665914 :         if (is_join(rel->op) && !is_outerjoin(rel->op) && !is_single(rel) && !list_empty(rel->exps) && !rel_is_ref(rel)) {
    5322      551608 :                 sql_rel *l = rel->l, *r = rel->r;
    5323             : 
    5324      551608 :                 if (is_left(r->op) && (is_select(l->op) || (is_join(l->op) && !is_outerjoin(l->op))) && !rel_is_ref(l) &&
    5325         952 :                                 !rel_is_ref(r)) {
    5326         952 :                         sql_rel *rl = r->l;
    5327         952 :                         sql_rel *rr = r->r;
    5328         952 :                         if (rel_is_ref(rl) || rel_is_ref(rr))
    5329             :                                 return rel;
    5330             :                         /* join exps should only include l and r.l */
    5331         952 :                         list *njexps = sa_list(v->sql->sa);
    5332        2320 :                         for(node *n = rel->exps->h; n; n = n->next) {
    5333        1368 :                                 sql_exp *je = n->data;
    5334             : 
    5335        1368 :                                 assert(je->type == e_cmp);
    5336        1368 :                                 if (je->f)
    5337             :                                         return rel;
    5338        1368 :                                 if ((rel_find_exp(l, je->l) && rel_find_exp(rl, je->r)) || (rel_find_exp(l, je->r) && rel_find_exp(rl, je->l))) {
    5339        1368 :                                         list_append(njexps, je);
    5340             :                                 } else {
    5341           0 :                                         return rel;
    5342             :                                 }
    5343             :                         }
    5344         952 :                         sql_rel *nl = rel_crossproduct(v->sql->sa, rel_dup(l), rl, rel->op);
    5345         952 :                         r->l = nl;
    5346         952 :                         nl->exps = njexps;
    5347         952 :                         rel_dup(r);
    5348         952 :                         rel_destroy(rel);
    5349             :                         rel = r;
    5350         952 :                         v->changes++;
    5351             :                 }
    5352             :         }
    5353             :         return rel;
    5354             : }
    5355             : 
    5356             : #define NO_PROJECTION_FOUND 0
    5357             : #define MAY_HAVE_DUPLICATE_NULLS 1
    5358             : #define ALL_VALUES_DISTINCT 2
    5359             : 
    5360             : static int
    5361     1053886 : find_projection_for_join2semi(sql_rel *rel)
    5362             : {
    5363     1053886 :         if (is_simple_project(rel->op) || is_groupby(rel->op) || is_inter(rel->op) || is_except(rel->op) || is_base(rel->op) || (is_union(rel->op) && need_distinct(rel))) {
    5364      543642 :                 if (rel->card < CARD_AGGR) /* const or groupby without group by exps */
    5365             :                         return ALL_VALUES_DISTINCT;
    5366      541106 :                 if (list_length(rel->exps) == 1) {
    5367       52336 :                         sql_exp *e = rel->exps->h->data;
    5368             :                         /* a single group by column in the projection list from a group by relation is guaranteed to be unique, but not an aggregate */
    5369       52336 :                         if (e->type == e_column) {
    5370       52292 :                                 sql_rel *res = NULL;
    5371             :                                 sql_exp *found = NULL;
    5372       52292 :                                 bool underjoin = false;
    5373             : 
    5374             :                                 /* if just one groupby column is projected or the relation needs distinct values and one column is projected or is a primary key, it will be distinct */
    5375       52292 :                                 if ((is_groupby(rel->op) && list_length(rel->r) == 1 && exps_find_exp(rel->r, e)) || (need_distinct(rel) && list_length(rel->exps) == 1))
    5376       48999 :                                         return ALL_VALUES_DISTINCT;
    5377        3447 :                                 if (is_unique(e))
    5378         151 :                                         return has_nil(e) ? MAY_HAVE_DUPLICATE_NULLS : ALL_VALUES_DISTINCT;
    5379             : 
    5380        4452 :                                 if ((is_simple_project(rel->op) || is_groupby(rel->op) || is_inter(rel->op) || is_except(rel->op)) &&
    5381        2312 :                                         (found = rel_find_exp_and_corresponding_rel(rel->l, e, &res, &underjoin)) && !underjoin) { /* grouping column on inner relation */
    5382        1106 :                                         if (need_distinct(res) && list_length(res->exps) == 1)
    5383             :                                                 return ALL_VALUES_DISTINCT;
    5384        1106 :                                         if (is_unique(found))
    5385           1 :                                                 return has_nil(e) ? MAY_HAVE_DUPLICATE_NULLS : ALL_VALUES_DISTINCT;
    5386        1105 :                                         if (found->type == e_column && found->card <= CARD_AGGR) {
    5387           2 :                                                 if (!is_groupby(res->op) && list_length(res->exps) != 1)
    5388             :                                                         return NO_PROJECTION_FOUND;
    5389           3 :                                                 for (node *n = res->exps->h ; n ; n = n->next) { /* must be the single column in the group by expression list */
    5390           2 :                                                         sql_exp *e = n->data;
    5391           2 :                                                         if (e != found && e->type == e_column)
    5392             :                                                                 return NO_PROJECTION_FOUND;
    5393             :                                                 }
    5394             :                                                 return ALL_VALUES_DISTINCT;
    5395             :                                         }
    5396             :                                 }
    5397             :                         }
    5398             :                 }
    5399             :         }
    5400             :         return NO_PROJECTION_FOUND;
    5401             : }
    5402             : 
    5403             : static sql_rel *
    5404     3129446 : find_candidate_join2semi(visitor *v, sql_rel *rel, bool *swap)
    5405             : {
    5406             :         /* generalize possibility : we need the visitor 'step' here */
    5407     3130119 :         if (rel_is_ref(rel)) /* if the join has multiple references, it's dangerous to convert it into a semijoin */
    5408             :                 return NULL;
    5409     2946940 :         if (rel->op == op_join && !list_empty(rel->exps)) {
    5410      528352 :                 sql_rel *l = rel->l, *r = rel->r;
    5411             :                 int foundr = NO_PROJECTION_FOUND, foundl = NO_PROJECTION_FOUND, found = NO_PROJECTION_FOUND;
    5412             :                 bool ok = false;
    5413             : 
    5414      528352 :                 foundr = find_projection_for_join2semi(r);
    5415      528352 :                 if (foundr < ALL_VALUES_DISTINCT)
    5416      525534 :                         foundl = find_projection_for_join2semi(l);
    5417      528352 :                 if (foundr && foundr > foundl) {
    5418        2821 :                         *swap = false;
    5419             :                         found = foundr;
    5420      525531 :                 } else if (foundl) {
    5421       48713 :                         *swap = true;
    5422             :                         found = foundl;
    5423             :                 }
    5424             : 
    5425       51534 :                 if (found > NO_PROJECTION_FOUND) {
    5426             :                         /* if all join expressions can be pushed down or have function calls, then it cannot be rewritten into a semijoin */
    5427      103070 :                         for (node *n=rel->exps->h; n && !ok; n = n->next) {
    5428       51536 :                                 sql_exp *e = n->data;
    5429             : 
    5430       51539 :                                 ok |= e->type == e_cmp && e->flag == cmp_equal && !exp_has_func(e) && !rel_rebind_exp(v->sql, l, e) && !rel_rebind_exp(v->sql, r, e) &&
    5431           3 :                                         (found == ALL_VALUES_DISTINCT || !is_semantics(e) || !has_nil((sql_exp *)e->l) || !has_nil((sql_exp *)e->r));
    5432             :                         }
    5433             :                 }
    5434             : 
    5435      528352 :                 if (ok)
    5436             :                         return rel;
    5437             :         }
    5438     2899168 :         if (is_join(rel->op) || is_semi(rel->op)) {
    5439             :                 sql_rel *c;
    5440             : 
    5441     1595426 :                 if ((c=find_candidate_join2semi(v, rel->l, swap)) != NULL ||
    5442      797176 :                     (c=find_candidate_join2semi(v, rel->r, swap)) != NULL)
    5443        1236 :                         return c;
    5444             :         }
    5445     2897932 :         if (is_topn(rel->op) || is_sample(rel->op))
    5446         673 :                 return find_candidate_join2semi(v, rel->l, swap);
    5447             :         return NULL;
    5448             : }
    5449             : 
    5450             : static int
    5451         698 : subrel_uses_exp_outside_subrel(sql_rel *rel, list *l, sql_rel *c)
    5452             : {
    5453         698 :         if (rel == c)
    5454             :                 return 0;
    5455             :         /* for subrel only expect joins (later possibly selects) */
    5456         309 :         if (is_join(rel->op) || is_semi(rel->op)) {
    5457         156 :                 if (exps_uses_any(rel->exps, l))
    5458             :                         return 1;
    5459         307 :                 if (subrel_uses_exp_outside_subrel(rel->l, l, c) ||
    5460         153 :                     subrel_uses_exp_outside_subrel(rel->r, l, c))
    5461           1 :                         return 1;
    5462             :         }
    5463         306 :         if (is_topn(rel->op) || is_sample(rel->op))
    5464           0 :                 return subrel_uses_exp_outside_subrel(rel->l, l, c);
    5465             :         return 0;
    5466             : }
    5467             : 
    5468             : static int
    5469       47772 : rel_uses_exp_outside_subrel(sql_rel *rel, list *l, sql_rel *c)
    5470             : {
    5471             :         /* for now we only expect sub relations of type project, selects (rel) or join/semi */
    5472       47772 :         if (is_simple_project(rel->op) || is_groupby(rel->op) || is_select(rel->op)) {
    5473       47772 :                 if (!list_empty(rel->exps) && exps_uses_any(rel->exps, l))
    5474             :                         return 1;
    5475         391 :                 if ((is_simple_project(rel->op) || is_groupby(rel->op)) && !list_empty(rel->r) && exps_uses_any(rel->r, l))
    5476             :                         return 1;
    5477         391 :                 if (rel->l)
    5478         391 :                         return subrel_uses_exp_outside_subrel(rel->l, l, c);
    5479             :         }
    5480           0 :         if (is_topn(rel->op) || is_sample(rel->op))
    5481           0 :                 return subrel_uses_exp_outside_subrel(rel->l, l, c);
    5482             :         return 1;
    5483             : }
    5484             : 
    5485             : static inline sql_rel *
    5486     4665914 : rel_join2semijoin(visitor *v, sql_rel *rel)
    5487             : {
    5488     4665914 :         if ((is_simple_project(rel->op) || is_groupby(rel->op)) && rel->l) {
    5489     1534020 :                 bool swap = false;
    5490             :                 sql_rel *l = rel->l;
    5491     1534020 :                 sql_rel *c = find_candidate_join2semi(v, l, &swap);
    5492             : 
    5493     1534020 :                 if (c) {
    5494             :                         /* 'p' is a project */
    5495       47772 :                         sql_rel *p = swap ? c->l : c->r;
    5496             : 
    5497             :                         /* now we need to check if ce is only used at the level of c */
    5498       47772 :                         if (!rel_uses_exp_outside_subrel(rel, p->exps, c)) {
    5499         389 :                                 c->op = op_semi;
    5500         389 :                                 if (swap) {
    5501           4 :                                         sql_rel *tmp = c->r;
    5502           4 :                                         c->r = c->l;
    5503           4 :                                         c->l = tmp;
    5504             :                                 }
    5505         389 :                                 v->changes++;
    5506             :                         }
    5507             :                 }
    5508             :         }
    5509     4665914 :         return rel;
    5510             : }
    5511             : 
    5512             : static int
    5513     3792636 : exp_is_rename(sql_exp *e)
    5514             : {
    5515     3792636 :         return (e->type == e_column);
    5516             : }
    5517             : 
    5518             : static int
    5519     2028575 : exp_is_useless_rename(sql_exp *e)
    5520             : {
    5521     4057150 :         return (e->type == e_column &&
    5522     2028575 :                         ((!e->l && !exp_relname(e)) ||
    5523     4056867 :                          (e->l && exp_relname(e) && strcmp(e->l, exp_relname(e)) == 0)) &&
    5524     1794397 :                         strcmp(e->r, exp_name(e)) == 0);
    5525             : }
    5526             : 
    5527             : static list *
    5528       10213 : rel_used_projections(mvc *sql, list *exps, list *users)
    5529             : {
    5530       10213 :         list *nexps = sa_list(sql->sa);
    5531       10213 :         bool *used = SA_ZNEW_ARRAY(sql->ta, bool, list_length(exps));
    5532             :         int i = 0;
    5533             : 
    5534       61928 :         for(node *n = users->h; n; n = n->next) {
    5535       51715 :                 sql_exp *e = n->data, *ne = NULL;
    5536       51715 :                 if ((e->l && (ne = exps_bind_column2(exps, e->l, e->r, NULL))) || (ne = exps_bind_column(exps, e->r, NULL, NULL, 1))) {
    5537       51715 :                         used[list_position(exps, ne)] = 1;
    5538             :                 }
    5539             :         }
    5540       60062 :         for(node *n = exps->h; n; n = n->next, i++) {
    5541       49849 :                 sql_exp *e = n->data;
    5542       49849 :                 if (is_intern(e) || used[i])
    5543       49849 :                         append(nexps, e);
    5544             :         }
    5545       10213 :         return nexps;
    5546             : }
    5547             : 
    5548             : /* move projects down with the goal op removing them completely (ie push renames/reduced lists into basetable)
    5549             :  * for some cases we can directly remove iff renames rename into same alias
    5550             :  * */
    5551             : static sql_rel *
    5552     5381801 : rel_push_project_down(visitor *v, sql_rel *rel)
    5553             : {
    5554             :         /* for now only push down renames */
    5555     5381801 :         if (v->depth > 1 && is_simple_project(rel->op) && !need_distinct(rel) && !rel_is_ref(rel) && rel->l && !rel->r &&
    5556     1177252 :                         v->parent &&
    5557     1167498 :                         !is_modify(v->parent->op) && !is_topn(v->parent->op) && !is_sample(v->parent->op) &&
    5558     1891516 :                         !is_ddl(v->parent->op) && !is_set(v->parent->op) &&
    5559      770524 :                         list_check_prop_all(rel->exps, (prop_check_func)&exp_is_rename)) {
    5560      402988 :                 sql_rel *l = rel->l;
    5561             : 
    5562      402988 :                 if (rel_is_ref(l))
    5563             :                         return rel;
    5564      340944 :                 if (is_basetable(l->op)) {
    5565       24634 :                         if (list_check_prop_all(rel->exps, (prop_check_func)&exp_is_useless_rename)) {
    5566             :                                 /* TODO reduce list (those in the project + internal) */
    5567       10213 :                                 rel->l = NULL;
    5568       10213 :                                 l->exps = rel_used_projections(v->sql, l->exps, rel->exps);
    5569       10213 :                                 rel_destroy(rel);
    5570       10213 :                                 v->changes++;
    5571       10213 :                                 return l;
    5572             :                         }
    5573             :                         return rel;
    5574      316310 :                 } else if (list_check_prop_all(rel->exps, (prop_check_func)&exp_is_useless_rename)) {
    5575       96353 :                         if ((is_project(l->op) && list_length(l->exps) == list_length(rel->exps)) ||
    5576       86499 :                                 ((v->parent && is_project(v->parent->op)) &&
    5577       40679 :                                  (is_set(l->op) || is_select(l->op) || is_join(l->op) || is_semi(l->op) || is_topn(l->op) || is_sample(l->op)))) {
    5578       43932 :                                 rel->l = NULL;
    5579       43932 :                                 rel_destroy(rel);
    5580       43932 :                                 v->changes++;
    5581       43932 :                                 return l;
    5582             :                         }
    5583             :                 }
    5584             :         }
    5585             :         return rel;
    5586             : }
    5587             : 
    5588             : static inline sql_rel *
    5589     3866011 : rel_push_project_down_union(visitor *v, sql_rel *rel)
    5590             : {
    5591             :         /* first remove distinct if already unique */
    5592     3866011 :         if (rel->op == op_project && need_distinct(rel) && rel->exps && exps_unique(v->sql, rel, rel->exps) && !have_nil(rel->exps)) {
    5593         104 :                 set_nodistinct(rel);
    5594         104 :                 if (exps_card(rel->exps) <= CARD_ATOM && rel->card > CARD_ATOM) /* if the projection just contains constants, then no topN is needed */
    5595          19 :                         rel->l = rel_topn(v->sql->sa, rel->l, append(sa_list(v->sql->sa), exp_atom_lng(v->sql->sa, 1)));
    5596         104 :                 v->changes++;
    5597             :         }
    5598             : 
    5599     3866011 :         if (rel->op == op_project && rel->l && rel->exps && !rel->r) {
    5600     1112244 :                 int need_distinct = need_distinct(rel);
    5601             :                 sql_rel *u = rel->l;
    5602             :                 sql_rel *p = rel;
    5603     1112244 :                 sql_rel *ul = u->l;
    5604     1112244 :                 sql_rel *ur = u->r;
    5605             : 
    5606     1112244 :                 if (!u || !is_union(u->op) || need_distinct(u) || !u->exps || rel_is_ref(u) || project_unsafe(rel,0))
    5607     1085366 :                         return rel;
    5608             :                 /* don't push project down union of single values */
    5609       26878 :                 if ((is_project(ul->op) && !ul->l) || (is_project(ur->op) && !ur->l))
    5610             :                         return rel;
    5611             : 
    5612       26865 :                 ul = rel_dup(ul);
    5613       26865 :                 ur = rel_dup(ur);
    5614             : 
    5615       26865 :                 if (!is_project(ul->op))
    5616         319 :                         ul = rel_project(v->sql->sa, ul,
    5617             :                                 rel_projections(v->sql, ul, NULL, 1, 1));
    5618       26865 :                 if (!is_project(ur->op))
    5619         799 :                         ur = rel_project(v->sql->sa, ur,
    5620             :                                 rel_projections(v->sql, ur, NULL, 1, 1));
    5621       26865 :                 need_distinct = (need_distinct &&
    5622           0 :                                 (!exps_unique(v->sql, ul, ul->exps) || have_nil(ul->exps) ||
    5623           0 :                                  !exps_unique(v->sql, ur, ur->exps) || have_nil(ur->exps)));
    5624       26865 :                 rel_rename_exps(v->sql, u->exps, ul->exps);
    5625       26865 :                 rel_rename_exps(v->sql, u->exps, ur->exps);
    5626             : 
    5627             :                 /* introduce projects under the set */
    5628       26865 :                 ul = rel_project(v->sql->sa, ul, NULL);
    5629       26865 :                 if (need_distinct)
    5630           0 :                         set_distinct(ul);
    5631       26865 :                 ur = rel_project(v->sql->sa, ur, NULL);
    5632       26865 :                 if (need_distinct)
    5633           0 :                         set_distinct(ur);
    5634             : 
    5635       26865 :                 ul->exps = exps_copy(v->sql, p->exps);
    5636       26865 :                 ur->exps = exps_copy(v->sql, p->exps);
    5637             : 
    5638       26865 :                 rel = rel_inplace_setop(v->sql, rel, ul, ur, op_union,
    5639             :                         rel_projections(v->sql, rel, NULL, 1, 1));
    5640       26865 :                 if (need_distinct)
    5641           0 :                         set_distinct(rel);
    5642       26865 :                 if (is_single(u))
    5643           0 :                         set_single(rel);
    5644       26865 :                 v->changes++;
    5645       26865 :                 rel->l = rel_merge_projects(v, rel->l);
    5646       26865 :                 rel->r = rel_merge_projects(v, rel->r);
    5647       26865 :                 return rel;
    5648             :         }
    5649             :         return rel;
    5650             : }
    5651             : 
    5652             : static int
    5653      280513 : sql_class_base_score(visitor *v, sql_column *c, sql_subtype *t, bool equality_based)
    5654             : {
    5655             :         int de;
    5656             : 
    5657      280513 :         if (!t)
    5658             :                 return 0;
    5659      280512 :         switch (ATOMstorage(t->type->localtype)) {
    5660             :                 case TYPE_bte:
    5661             :                         return 150 - 8;
    5662       12261 :                 case TYPE_sht:
    5663       12261 :                         return 150 - 16;
    5664       90372 :                 case TYPE_int:
    5665       90372 :                         return 150 - 32;
    5666        2267 :                 case TYPE_void:
    5667             :                 case TYPE_lng:
    5668        2267 :                         return 150 - 64;
    5669        1057 :                 case TYPE_uuid:
    5670             : #ifdef HAVE_HGE
    5671             :                 case TYPE_hge:
    5672             : #endif
    5673        1057 :                         return 150 - 128;
    5674          12 :                 case TYPE_flt:
    5675          12 :                         return 75 - 24;
    5676         232 :                 case TYPE_dbl:
    5677         232 :                         return 75 - 53;
    5678       70429 :                 default: {
    5679       70429 :                         if (equality_based && c && v->storage_based_opt && (de = mvc_is_duplicate_eliminated(v->sql, c)))
    5680        5905 :                                 return 150 - de * 8;
    5681             :                         /* strings and blobs not duplicate eliminated don't get any points here */
    5682             :                         return 0;
    5683             :                 }
    5684             :         }
    5685             : }
    5686             : 
    5687             : /* Compute the efficiency of using this expression earl y in a group by list */
    5688             : static int
    5689      127469 : score_gbe(visitor *v, sql_rel *rel, sql_exp *e)
    5690             : {
    5691             :         int res = 0;
    5692      127469 :         sql_subtype *t = exp_subtype(e);
    5693      127469 :         sql_column *c = exp_find_column(rel, e, -2);
    5694             : 
    5695      127469 :         if (e->card == CARD_ATOM) /* constants are trivial to group */
    5696             :                 res += 1000;
    5697             :         /* can we find out if the underlying table is sorted */
    5698      127469 :         if (is_unique(e) || find_prop(e->p, PROP_HASHCOL) || (c && v->storage_based_opt && mvc_is_unique(v->sql, c))) /* distinct columns */
    5699        8359 :                 res += 700;
    5700      127469 :         if (c && v->storage_based_opt && mvc_is_sorted(v->sql, c))
    5701        7745 :                 res += 500;
    5702      127469 :         if (find_prop(e->p, PROP_HASHIDX)) /* has hash index */
    5703           0 :                 res += 200;
    5704             : 
    5705             :         /* prefer the shorter var types over the longer ones */
    5706      127469 :         res += sql_class_base_score(v, c, t, true); /* smaller the type, better */
    5707      127469 :         return res;
    5708             : }
    5709             : 
    5710             : /* reorder group by expressions */
    5711             : static inline sql_rel *
    5712      200583 : rel_groupby_order(visitor *v, sql_rel *rel)
    5713             : {
    5714             :         int *scores = NULL;
    5715             :         sql_exp **exps = NULL;
    5716             : 
    5717      200583 :         if (is_groupby(rel->op) && list_length(rel->r) > 1) {
    5718             :                 node *n;
    5719       55539 :                 list *gbe = rel->r;
    5720       55539 :                 int i, ngbe = list_length(gbe);
    5721       55539 :                 scores = SA_NEW_ARRAY(v->sql->ta, int, ngbe);
    5722       55539 :                 exps = SA_NEW_ARRAY(v->sql->ta, sql_exp*, ngbe);
    5723             : 
    5724             :                 /* first sorting step, give priority for integers and sorted columns */
    5725      183008 :                 for (i = 0, n = gbe->h; n; i++, n = n->next) {
    5726      127469 :                         exps[i] = n->data;
    5727      127469 :                         scores[i] = score_gbe(v, rel, exps[i]);
    5728             :                 }
    5729       55539 :                 GDKqsort(scores, exps, NULL, ngbe, sizeof(int), sizeof(void *), TYPE_int, true, true);
    5730             : 
    5731             :                 /* second sorting step, give priority to strings with lower number of digits */
    5732      104052 :                 for (i = ngbe - 1; i && !scores[i]; i--); /* find expressions with no score from the first round */
    5733       55539 :                 if (scores[i])
    5734       54196 :                         i++;
    5735       55539 :                 if (ngbe - i > 1) {
    5736       10934 :                         for (int j = i; j < ngbe; j++) {
    5737        8304 :                                 sql_subtype *t = exp_subtype(exps[j]);
    5738        8304 :                                 scores[j] = t ? t->digits : 0;
    5739             :                         }
    5740             :                         /* the less number of digits the better, order ascending */
    5741        2630 :                         GDKqsort(scores + i, exps + i, NULL, ngbe - i, sizeof(int), sizeof(void *), TYPE_int, false, true);
    5742             :                 }
    5743             : 
    5744      183008 :                 for (i = 0, n = gbe->h; n; i++, n = n->next)
    5745      127469 :                         n->data = exps[i];
    5746             :         }
    5747             : 
    5748      200583 :         return rel;
    5749             : }
    5750             : 
    5751             : /* reduce group by expressions based on pkey info
    5752             :  *
    5753             :  * The reduced group by and (derived) aggr expressions are restored via
    5754             :  * extra (new) aggregate columns.
    5755             :  */
    5756             : static inline sql_rel *
    5757      200583 : rel_reduce_groupby_exps(visitor *v, sql_rel *rel)
    5758             : {
    5759      200583 :         list *gbe = rel->r;
    5760             : 
    5761      200583 :         if (is_groupby(rel->op) && rel->r && !rel_is_ref(rel) && list_length(gbe)) {
    5762             :                 node *n, *m;
    5763      125911 :                 int k, j, i, ngbe = list_length(gbe);
    5764      125911 :                 int8_t *scores = SA_NEW_ARRAY(v->sql->ta, int8_t, ngbe);
    5765             :                 sql_column *c;
    5766      125911 :                 sql_table **tbls = SA_NEW_ARRAY(v->sql->ta, sql_table*, ngbe);
    5767      125911 :                 sql_rel **bts = SA_NEW_ARRAY(v->sql->ta, sql_rel*, ngbe), *bt = NULL;
    5768             : 
    5769      125911 :                 gbe = rel->r;
    5770      295358 :                 for (k = 0, i = 0, n = gbe->h; n; n = n->next, k++) {
    5771      169447 :                         sql_exp *e = n->data;
    5772             : 
    5773      169447 :                         c = exp_find_column_(rel, e, -2, &bt);
    5774      169447 :                         if (c) {
    5775      163193 :                                 for(j = 0; j < i; j++)
    5776       35369 :                                         if (c->t == tbls[j] && bts[j] == bt)
    5777             :                                                 break;
    5778      146743 :                                 tbls[j] = c->t;
    5779      146743 :                                 bts[j] = bt;
    5780      146743 :                                 i += (j == i);
    5781             :                         }
    5782             :                 }
    5783      125911 :                 if (i) { /* forall tables find pkey and
    5784             :                                 remove useless other columns */
    5785             :                         /* TODO also remove group by columns which are related to
    5786             :                          * the other columns using a foreign-key join (n->1), ie 1
    5787             :                          * on the to be removed side.
    5788             :                          */
    5789      248196 :                         for(j = 0; j < i; j++) {
    5790             :                                 int l, nr = 0, cnr = 0;
    5791             : 
    5792      127803 :                                 k = list_length(gbe);
    5793      127803 :                                 memset(scores, 0, list_length(gbe));
    5794      127803 :                                 if (tbls[j]->pkey) {
    5795       15001 :                                         for (l = 0, n = gbe->h; l < k && n; l++, n = n->next) {
    5796             :                                                 fcmp cmp = (fcmp)&kc_column_cmp;
    5797       11242 :                                                 sql_exp *e = n->data;
    5798             : 
    5799       11242 :                                                 c = exp_find_column_(rel, e, -2, &bt);
    5800       17154 :                                                 if (c && c->t == tbls[j] && bts[j] == bt &&
    5801        5912 :                                                     list_find(tbls[j]->pkey->k.columns, c, cmp) != NULL) {
    5802         807 :                                                         scores[l] = 1;
    5803         807 :                                                         nr ++;
    5804       10435 :                                                 } else if (c && c->t == tbls[j] && bts[j] == bt) {
    5805             :                                                         /* Okay we can cleanup a group by column */
    5806        5105 :                                                         scores[l] = -1;
    5807        5105 :                                                         cnr ++;
    5808             :                                                 }
    5809             :                                         }
    5810             :                                 }
    5811        3759 :                                 if (nr) {
    5812         799 :                                         int all = (list_length(tbls[j]->pkey->k.columns) == nr);
    5813         799 :                                         sql_kc *kc = tbls[j]->pkey->k.columns->h->data;
    5814             : 
    5815         799 :                                         c = kc->c;
    5816        2088 :                                         for (l = 0, n = gbe->h; l < k && n; l++, n = n->next) {
    5817        1289 :                                                 sql_exp *e = n->data;
    5818             : 
    5819             :                                                 /* pkey based group by */
    5820        1435 :                                                 if (scores[l] == 1 && ((all ||
    5821             :                                                    /* first of key */
    5822         931 :                                                    (c == exp_find_column(rel, e, -2))) && !find_prop(e->p, PROP_HASHCOL)))
    5823          10 :                                                         e->p = prop_create(v->sql->sa, PROP_HASHCOL, e->p);
    5824             :                                         }
    5825        2914 :                                         for (m = rel->exps->h; m; m = m->next ){
    5826        2115 :                                                 sql_exp *e = m->data;
    5827             : 
    5828        7017 :                                                 for (l = 0, n = gbe->h; l < k && n; l++, n = n->next) {
    5829        4909 :                                                         sql_exp *gb = n->data;
    5830             : 
    5831             :                                                         /* pkey based group by */
    5832        4909 :                                                         if (scores[l] == 1 && exp_match_exp(e,gb) && find_prop(gb->p, PROP_HASHCOL) && !find_prop(e->p, PROP_HASHCOL)) {
    5833           7 :                                                                 e->p = prop_create(v->sql->sa, PROP_HASHCOL, e->p);
    5834           7 :                                                                 break;
    5835             :                                                         }
    5836             : 
    5837             :                                                 }
    5838             :                                         }
    5839             :                                 }
    5840      127803 :                                 if (cnr && nr && list_length(tbls[j]->pkey->k.columns) == nr) {
    5841         108 :                                         list *ngbe = new_exp_list(v->sql->sa);
    5842             : 
    5843         392 :                                         for (l = 0, n = gbe->h; l < k && n; l++, n = n->next) {
    5844         284 :                                                 sql_exp *e = n->data;
    5845             : 
    5846             :                                                 /* keep the group by columns which form a primary key
    5847             :                                                  * of this table. And those unrelated to this table. */
    5848         284 :                                                 if (scores[l] != -1)
    5849         139 :                                                         append(ngbe, e);
    5850             :                                         }
    5851         108 :                                         rel->r = ngbe;
    5852             :                                         /* rewrite gbe and aggr, in the aggr list */
    5853         410 :                                         for (m = rel->exps->h; m; m = m->next ){
    5854         302 :                                                 sql_exp *e = m->data;
    5855             :                                                 int fnd = 0;
    5856             : 
    5857        1398 :                                                 for (l = 0, n = gbe->h; l < k && n && !fnd; l++, n = n->next) {
    5858        1096 :                                                         sql_exp *gb = n->data;
    5859             : 
    5860        1096 :                                                         if (scores[l] == -1 && exp_refers(gb, e)) {
    5861         145 :                                                                 sql_exp *rs = exp_column(v->sql->sa, gb->l?gb->l:exp_relname(gb), gb->r?gb->r:exp_name(gb), exp_subtype(gb), rel->card, has_nil(gb), is_unique(gb), is_intern(gb));
    5862         145 :                                                                 exp_setname(v->sql->sa, rs, exp_find_rel_name(e), exp_name(e));
    5863             :                                                                 e = rs;
    5864             :                                                                 fnd = 1;
    5865             :                                                         }
    5866             :                                                 }
    5867         302 :                                                 m->data = e;
    5868             :                                         }
    5869             :                                         /* new reduced aggr expression list */
    5870         108 :                                         assert(list_length(rel->exps)>0);
    5871             :                                         /* only one reduction at a time */
    5872         108 :                                         list_hash_clear(rel->exps);
    5873         108 :                                         v->changes++;
    5874         108 :                                         return rel;
    5875             :                                 }
    5876      127695 :                                 gbe = rel->r;
    5877             :                         }
    5878             :                 }
    5879             :         }
    5880             :         /* remove constants from group by list */
    5881      200475 :         if (is_groupby(rel->op) && rel->r && !rel_is_ref(rel)) {
    5882             :                 int i;
    5883             :                 node *n;
    5884             : 
    5885      296971 :                 for (i = 0, n = gbe->h; n; n = n->next) {
    5886      169163 :                         sql_exp *e = n->data;
    5887             : 
    5888      169163 :                         if (exp_is_atom(e))
    5889          65 :                                 i++;
    5890             :                 }
    5891      127808 :                 if (i) {
    5892          50 :                         list *ngbe = new_exp_list(v->sql->sa);
    5893          50 :                         list *dgbe = new_exp_list(v->sql->sa);
    5894             : 
    5895         116 :                         for (n = gbe->h; n; n = n->next) {
    5896          66 :                                 sql_exp *e = n->data;
    5897             : 
    5898          66 :                                 if (!exp_is_atom(e))
    5899           1 :                                         append(ngbe, e);
    5900             :                                 /* we need at least one gbe */
    5901          65 :                                 else if (!n->next && list_empty(ngbe))
    5902          49 :                                         append(ngbe, e);
    5903             :                                 else
    5904          16 :                                         append(dgbe, e);
    5905             :                         }
    5906          50 :                         rel->r = ngbe;
    5907          50 :                         if (!list_empty(dgbe)) {
    5908             :                                 /* use atom's directly in the aggr expr list */
    5909             : 
    5910          38 :                                 for (n = rel->exps->h; n; n = n->next) {
    5911          31 :                                         sql_exp *e = n->data, *ne = NULL;
    5912             : 
    5913          31 :                                         if (e->type == e_column) {
    5914          23 :                                                 if (e->l)
    5915           8 :                                                         ne = exps_bind_column2(dgbe, e->l, e->r, NULL);
    5916             :                                                 else
    5917          15 :                                                         ne = exps_bind_column(dgbe, e->r, NULL, NULL, 1);
    5918          23 :                                                 if (ne) {
    5919          16 :                                                         ne = exp_copy(v->sql, ne);
    5920          16 :                                                         exp_prop_alias(v->sql->sa, ne, e);
    5921             :                                                         e = ne;
    5922             :                                                 }
    5923             :                                         }
    5924          31 :                                         n->data = e;
    5925             :                                 }
    5926           7 :                                 list_hash_clear(rel->exps);
    5927           7 :                                 v->changes++;
    5928             :                         }
    5929             :                 }
    5930             :         }
    5931             :         return rel;
    5932             : }
    5933             : 
    5934             : #if 0
    5935             : static sql_rel *
    5936             : rel_groupby_distinct2(visitor *v, sql_rel *rel)
    5937             : {
    5938             :         list *ngbes = sa_list(v->sql->sa), *gbes, *naggrs = sa_list(v->sql->sa), *aggrs = sa_list(v->sql->sa);
    5939             :         sql_rel *l;
    5940             :         node *n;
    5941             : 
    5942             :         gbes = rel->r;
    5943             :         if (!gbes)
    5944             :                 return rel;
    5945             : 
    5946             :         /* check if each aggr is, rewritable (max,min,sum,count)
    5947             :          *                        and only has one argument */
    5948             :         for (n = rel->exps->h; n; n = n->next) {
    5949             :                 sql_exp *e = n->data;
    5950             :                 sql_subfunc *af = e->f;
    5951             : 
    5952             :                 if (e->type == e_aggr &&
    5953             :                    (strcmp(af->func->base.name, "sum") &&
    5954             :                      strcmp(af->func->base.name, "count") &&
    5955             :                      strcmp(af->func->base.name, "min") &&
    5956             :                      strcmp(af->func->base.name, "max")))
    5957             :                         return rel;
    5958             :         }
    5959             : 
    5960             :         for (n = gbes->h; n; n = n->next) {
    5961             :                 sql_exp *e = n->data;
    5962             : 
    5963             :                 e = exp_column(v->sql->sa, exp_find_rel_name(e), exp_name(e), exp_subtype(e), e->card, has_nil(e), is_unique(e), is_intern(e));
    5964             :                 append(ngbes, e);
    5965             :         }
    5966             : 
    5967             :         /* 1 for each aggr(distinct v) add the attribute expression v to gbes and aggrs list
    5968             :          * 2 for each aggr(z) add aggr_phase2('z') to the naggrs list
    5969             :          * 3 for each group by col, add also to the naggrs list
    5970             :          * */
    5971             :         for (n = rel->exps->h; n; n = n->next) {
    5972             :                 sql_exp *e = n->data;
    5973             : 
    5974             :                 if (e->type == e_aggr && need_distinct(e)) { /* 1 */
    5975             :                         /* need column expression */
    5976             :                         list *args = e->l;
    5977             :                         sql_exp *v = args->h->data;
    5978             :                         append(gbes, v);
    5979             :                         if (!exp_name(v))
    5980             :                                 exp_label(v->sql->sa, v, ++v->sql->label);
    5981             :                         v = exp_column(v->sql->sa, exp_find_rel_name(v), exp_name(v), exp_subtype(v), v->card, has_nil(v), is_unique(v), is_intern(v));
    5982             :                         append(aggrs, v);
    5983             :                         v = exp_aggr1(v->sql->sa, v, e->f, need_distinct(e), 1, e->card, 1);
    5984             :                         exp_setname(v->sql->sa, v, exp_find_rel_name(e), exp_name(e));
    5985             :                         append(naggrs, v);
    5986             :                 } else if (e->type == e_aggr && !need_distinct(e)) {
    5987             :                         sql_exp *v;
    5988             :                         sql_subfunc *f = e->f;
    5989             :                         int cnt = exp_aggr_is_count(e);
    5990             :                         sql_subfunc *a = sql_bind_func(v->sql, "sys", (cnt)?"sum":f->func->base.name, exp_subtype(e), NULL, F_AGGR);
    5991             : 
    5992             :                         append(aggrs, e);
    5993             :                         if (!exp_name(e))
    5994             :                                 exp_label(v->sql->sa, e, ++v->sql->label);
    5995             :                         set_has_nil(e);
    5996             :                         v = exp_column(v->sql->sa, exp_find_rel_name(e), exp_name(e), exp_subtype(e), e->card, has_nil(e), is_unique(e), is_intern(e));
    5997             :                         v = exp_aggr1(v->sql->sa, v, a, 0, 1, e->card, 1);
    5998             :                         if (cnt)
    5999             :                                 set_zero_if_empty(v);
    6000             :                         exp_setname(v->sql->sa, v, exp_find_rel_name(e), exp_name(e));
    6001             :                         append(naggrs, v);
    6002             :                 } else { /* group by col */
    6003             :                         if (list_find_exp(gbes, e) || !list_find_exp(naggrs, e)) {
    6004             :                                 append(aggrs, e);
    6005             : 
    6006             :                                 e = exp_column(v->sql->sa, exp_find_rel_name(e), exp_name(e), exp_subtype(e), e->card, has_nil(e), is_unique(e), is_intern(e));
    6007             :                         }
    6008             :                         append(naggrs, e);
    6009             :                 }
    6010             :         }
    6011             : 
    6012             :         l = rel->l = rel_groupby(v->sql, rel->l, gbes);
    6013             :         l->exps = aggrs;
    6014             :         rel->r = ngbes;
    6015             :         rel->exps = naggrs;
    6016             :         v->changes++;
    6017             :         return rel;
    6018             : }
    6019             : #endif
    6020             : 
    6021             : /* Rewrite group by expressions with distinct
    6022             :  *
    6023             :  * ie select a, count(distinct b) from c where ... groupby a;
    6024             :  * No other aggregations should be present
    6025             :  *
    6026             :  * Rewrite the more general case, good for parallel execution
    6027             :  *
    6028             :  * groupby(R) [e,f] [ aggr1 a distinct, aggr2 b distinct, aggr3 c, aggr4 d]
    6029             :  *
    6030             :  * into
    6031             :  *
    6032             :  * groupby(
    6033             :  *      groupby(R) [e,f,a,b] [ a, b, aggr3 c, aggr4 d]
    6034             :  * ) [e,f]( aggr1 a distinct, aggr2 b distinct, aggr3_phase2 c, aggr4_phase2 d)
    6035             :  */
    6036             : static inline sql_rel *
    6037      200583 : rel_groupby_distinct(visitor *v, sql_rel *rel)
    6038             : {
    6039             :         node *n;
    6040             : 
    6041      200583 :         if (is_groupby(rel->op)) {
    6042      200583 :                 sql_rel *l = rel->l;
    6043      200583 :                 if (!l || is_groupby(l->op))
    6044             :                         return rel;
    6045             :         }
    6046      200342 :         if (is_groupby(rel->op) && rel->r && !rel_is_ref(rel)) {
    6047             :                 int nr = 0, anr = 0;
    6048             :                 list *gbe, *ngbe, *arg, *exps, *nexps;
    6049             :                 sql_exp *distinct = NULL, *darg, *found;
    6050             :                 sql_rel *l = NULL;
    6051             : 
    6052      428074 :                 for (n=rel->exps->h; n && nr <= 2; n = n->next) {
    6053      300319 :                         sql_exp *e = n->data;
    6054      300319 :                         if (need_distinct(e)) {
    6055             :                                 distinct = n->data;
    6056        3659 :                                 nr++;
    6057             :                         }
    6058      300319 :                         anr += is_aggr(e->type);
    6059             :                 }
    6060      127755 :                 if (nr < 1 || distinct->type != e_aggr)
    6061             :                         return rel;
    6062        2110 :                 if (nr > 1 || anr > nr)
    6063             :                         return rel;//rel_groupby_distinct2(v, rel);
    6064         122 :                 arg = distinct->l;
    6065         122 :                 if (list_length(arg) != 1 || list_length(rel->r) + nr != list_length(rel->exps))
    6066           2 :                         return rel;
    6067             : 
    6068         120 :                 gbe = rel->r;
    6069         120 :                 ngbe = sa_list(v->sql->sa);
    6070         120 :                 exps = sa_list(v->sql->sa);
    6071         120 :                 nexps = sa_list(v->sql->sa);
    6072         382 :                 for (n=rel->exps->h; n; n = n->next) {
    6073         262 :                         sql_exp *e = n->data;
    6074         262 :                         if (e != distinct) {
    6075         142 :                                 if (e->type == e_aggr) { /* copy the arguments to the aggregate */
    6076           0 :                                         list *args = e->l;
    6077           0 :                                         if (args) {
    6078           0 :                                                 for (node *n = args->h ; n ; n = n->next) {
    6079           0 :                                                         sql_exp *e = n->data;
    6080           0 :                                                         list_append(ngbe, exp_copy(v->sql, e));
    6081           0 :                                                         list_append(exps, exp_copy(v->sql, e));
    6082             :                                                 }
    6083             :                                         }
    6084             :                                 } else {
    6085         142 :                                         e = exp_ref(v->sql, e);
    6086         142 :                                         append(ngbe, e);
    6087         142 :                                         append(exps, e);
    6088             :                                 }
    6089         142 :                                 if (e->type == e_aggr) /* aggregates must be copied */
    6090           0 :                                         e = exp_copy(v->sql, e);
    6091             :                                 else
    6092         142 :                                         e = exp_ref(v->sql, e);
    6093         142 :                                 append(nexps, e);
    6094             :                         }
    6095             :                 }
    6096             : 
    6097         120 :                 darg = arg->h->data;
    6098         120 :                 if ((found = exps_find_exp(gbe, darg))) { /* first find if the aggregate argument already exists in the grouping list */
    6099           6 :                         darg = exp_ref(v->sql, found);
    6100             :                 } else {
    6101         114 :                         list_append(gbe, darg = exp_copy(v->sql, darg));
    6102         114 :                         exp_label(v->sql->sa, darg, ++v->sql->label);
    6103         114 :                         darg = exp_ref(v->sql, darg);
    6104             :                 }
    6105         120 :                 list_append(exps, darg);
    6106         120 :                 darg = exp_ref(v->sql, darg);
    6107         120 :                 arg->h->data = darg;
    6108         120 :                 l = rel->l = rel_groupby(v->sql, rel->l, gbe);
    6109         120 :                 l->exps = exps;
    6110         120 :                 set_processed(l);
    6111         120 :                 rel->r = ngbe;
    6112         120 :                 rel->exps = nexps;
    6113         120 :                 set_nodistinct(distinct);
    6114         120 :                 append(nexps, distinct);
    6115         120 :                 v->changes++;
    6116             :         }
    6117             :         return rel;
    6118             : }
    6119             : 
    6120             : static sql_exp *split_aggr_and_project(mvc *sql, list *aexps, sql_exp *e);
    6121             : 
    6122             : static void
    6123           8 : list_split_aggr_and_project(mvc *sql, list *aexps, list *exps)
    6124             : {
    6125           8 :         if (list_empty(exps))
    6126             :                 return ;
    6127          24 :         for(node *n = exps->h; n; n = n->next)
    6128          16 :                 n->data = split_aggr_and_project(sql, aexps, n->data);
    6129             : }
    6130             : 
    6131             : static sql_exp *
    6132          19 : split_aggr_and_project(mvc *sql, list *aexps, sql_exp *e)
    6133             : {
    6134          19 :         switch(e->type) {
    6135           0 :         case e_aggr:
    6136             :                 /* add to the aggrs */
    6137           0 :                 if (!exp_name(e))
    6138           0 :                         exp_label(sql->sa, e, ++sql->label);
    6139           0 :                 list_append(aexps, e);
    6140           0 :                 return exp_ref(sql, e);
    6141             :         case e_cmp:
    6142             :                 /* e_cmp's shouldn't exist in an aggr expression list */
    6143           0 :                 assert(0);
    6144             :         case e_convert:
    6145           3 :                 e->l = split_aggr_and_project(sql, aexps, e->l);
    6146           3 :                 return e;
    6147           3 :         case e_func:
    6148           3 :                 list_split_aggr_and_project(sql, aexps, e->l);
    6149           3 :                 return e;
    6150             :         case e_atom:
    6151             :         case e_column: /* constants and columns shouldn't be rewriten */
    6152             :         case e_psm:
    6153             :                 return e;
    6154             :         }
    6155           0 :         return NULL;
    6156             : }
    6157             : 
    6158             : static sql_exp *
    6159             : exp_use_consts(mvc *sql, sql_exp *e, list *consts);
    6160             : 
    6161             : static list *
    6162             : exps_use_consts(mvc *sql, list *exps, list *consts)
    6163             : {
    6164             :         if (list_empty(exps))
    6165             :                 return exps;
    6166             :         for(node *n = exps->h; n; n = n->next) {
    6167             :                 sql_exp *arg = n->data, *narg = NULL;
    6168             : 
    6169             :                 narg = exp_use_consts(sql, arg, consts);
    6170             :                 if (!narg)
    6171             :                         return NULL;
    6172             :                 narg = exp_propagate(sql->sa, narg, arg);
    6173             :                 n->data = narg;
    6174             :         }
    6175             :         return exps;
    6176             : }
    6177             : 
    6178             : static sql_exp *
    6179             : exp_use_consts(mvc *sql, sql_exp *e, list *consts)
    6180             : {
    6181             :         sql_exp *ne = NULL, *l, *r, *r2;
    6182             : 
    6183             :         switch(e->type) {
    6184             :         case e_column:
    6185             :                 if (e->l)
    6186             :                         ne = exps_bind_column2(consts, e->l, e->r, NULL);
    6187             :                 if (!ne && !e->l)
    6188             :                         ne = exps_bind_column(consts, e->r, NULL, NULL, 1);
    6189             :                 if (!ne)
    6190             :                         return e;
    6191             :                 return ne;
    6192             :         case e_cmp:
    6193             :                 if (e->flag == cmp_or || e->flag == cmp_filter) {
    6194             :                         list *l = exps_use_consts(sql, e->l, consts);
    6195             :                         list *r = exps_use_consts(sql, e->r, consts);
    6196             : 
    6197             :                         if (!l || !r)
    6198             :                                 return NULL;
    6199             :                         if (e->flag == cmp_filter)
    6200             :                                 return exp_filter(sql->sa, l, r, e->f, is_anti(e));
    6201             :                         return exp_or(sql->sa, l, r, is_anti(e));
    6202             :                 } else if (e->flag == cmp_in || e->flag == cmp_notin) {
    6203             :                         sql_exp *l = exp_use_consts(sql, e->l, consts);
    6204             :                         list *r = exps_use_consts(sql, e->r, consts);
    6205             : 
    6206             :                         if (!l || !r)
    6207             :                                 return NULL;
    6208             :                         return exp_in(sql->sa, l, r, e->flag);
    6209             :                 } else {
    6210             :                         l = exp_use_consts(sql, e->l, consts);
    6211             :                         r = exp_use_consts(sql, e->r, consts);
    6212             :                         if (e->f) {
    6213             :                                 r2 = exp_use_consts(sql, e->f, consts);
    6214             :                                 if (l && r && r2)
    6215             :                                         ne = exp_compare2(sql->sa, l, r, r2, e->flag, is_symmetric(e));
    6216             :                         } else if (l && r) {
    6217             :                                 ne = exp_compare(sql->sa, l, r, e->flag);
    6218             :                         }
    6219             :                 }
    6220             :                 if (!ne)
    6221             :                         return NULL;
    6222             :                 return exp_propagate(sql->sa, ne, e);
    6223             :         case e_convert:
    6224             :                 l = exp_use_consts(sql, e->l, consts);
    6225             :                 if (l)
    6226             :                         return exp_convert(sql->sa, l, exp_fromtype(e), exp_totype(e));
    6227             :                 return NULL;
    6228             :         case e_aggr:
    6229             :         case e_func: {
    6230             :                 list *l = e->l, *nl = NULL;
    6231             : 
    6232             :                 if (!list_empty(l)) {
    6233             :                         nl = exps_use_consts(sql, l, consts);
    6234             :                         if (!nl)
    6235             :                                 return NULL;
    6236             :                 }
    6237             :                 if (e->type == e_func)
    6238             :                         return exp_op(sql->sa, nl, e->f);
    6239             :                 else
    6240             :                         return exp_aggr(sql->sa, nl, e->f, need_distinct(e), need_no_nil(e), e->card, has_nil(e));
    6241             :         }
    6242             :         case e_atom: {
    6243             :                 list *l = e->f, *nl = NULL;
    6244             : 
    6245             :                 if (!list_empty(l)) {
    6246             :                         nl = exps_use_consts(sql, l, consts);
    6247             :                         if (!nl)
    6248             :                                 return NULL;
    6249             :                         return exp_values(sql->sa, nl);
    6250             :                 } else {
    6251             :                         return exp_copy(sql, e);
    6252             :                 }
    6253             :         }
    6254             :         case e_psm:
    6255             :                 return e;
    6256             :         }
    6257             :         return NULL;
    6258             : }
    6259             : 
    6260             : static list *
    6261             : exps_remove_dictexps(list *exps, sql_rel *r)
    6262             : {
    6263             :         if (list_empty(exps))
    6264             :                 return exps;
    6265             :         for (node *n = exps->h; n;) {
    6266             :                 node *next = n->next;
    6267             :                 sql_exp *arg = n->data;
    6268             : 
    6269             :                 if (list_find_exp(r->exps, arg->l) || list_find_exp(r->exps, arg->r))
    6270             :                         list_remove_node(exps, NULL, n);
    6271             :                 n = next;
    6272             :         }
    6273             :         return exps;
    6274             : }
    6275             : 
    6276             : static sql_rel *
    6277             : rel_remove_join(visitor *v, sql_rel *rel)
    6278             : {
    6279             :         if (is_join(rel->op) && !is_outerjoin(rel->op)) {
    6280             :                 sql_rel *l = rel->l;
    6281             :                 sql_rel *r = rel->r;
    6282             :                 int lconst = 0, rconst = 0;
    6283             : 
    6284             :                 if (!l || rel_is_ref(l) || !r || rel_is_ref(r) ||
    6285             :                    (l->op != op_project && r->op != op_project))
    6286             :                         return rel;
    6287             :                 if (l->op == op_project && exps_are_atoms(l->exps))
    6288             :                         lconst = 1;
    6289             :                 if (r->op == op_project && exps_are_atoms(r->exps))
    6290             :                         rconst = 1;
    6291             :                 if (lconst || rconst) {
    6292             :                         v->changes++;
    6293             :                         /* use constant (instead of alias) in expressions */
    6294             :                         if (lconst) {
    6295             :                                 sql_rel *s = l;
    6296             :                                 l = r;
    6297             :                                 r = s;
    6298             :                         }
    6299             :                         rel->exps = exps_use_consts(v->sql, rel->exps, r->exps);
    6300             :                         /* change into select */
    6301             :                         rel->op = op_select;
    6302             :                         rel->l = l;
    6303             :                         rel->r = NULL;
    6304             :                         /* wrap in a project including, the constant columns */
    6305             :                         rel = rel_project(v->sql->sa, rel, rel_projections(v->sql, l, NULL, 1, 1));
    6306             :                         list_merge(rel->exps, r->exps, (fdup)NULL);
    6307             :                 }
    6308             :         }
    6309             :         if (is_join(rel->op)) {
    6310             :                 sql_rel *l = rel->l;
    6311             :                 sql_rel *r = rel->r;
    6312             :                 int ldict = 0, rdict = 0;
    6313             : 
    6314             :                 if (!l || rel_is_ref(l) || !r || rel_is_ref(r) ||
    6315             :                    (l->op != op_basetable && r->op != op_basetable))
    6316             :                         return rel;
    6317             :                 /* check if dict (last column) isn't used, one column only */
    6318             :                 if (l->op == op_basetable && !l->l && list_length(l->exps) <= 1)
    6319             :                         ldict = 1;
    6320             :                 if (r->op == op_basetable && !r->l && list_length(r->exps) <= 1)
    6321             :                         rdict = 1;
    6322             :                 if (!ldict && !rdict)
    6323             :                         return rel;
    6324             :                 v->changes++;
    6325             : 
    6326             :                 assert(0);
    6327             :                 if (ldict) {
    6328             :                         sql_rel *s = l;
    6329             :                         l = r;
    6330             :                         r = s;
    6331             :                 }
    6332             :                 rel->exps = exps_remove_dictexps(rel->exps, r);
    6333             :                 /* change into select */
    6334             :                 rel->op = op_select;
    6335             :                 rel->l = l;
    6336             :                 rel->r = NULL;
    6337             :                 /* wrap in a project including, the dict/index columns */
    6338             :                 rel = rel_project(v->sql->sa, rel, rel_projections(v->sql, l, NULL, 1, 1));
    6339             :                 list_merge(rel->exps, r->exps, (fdup)NULL);
    6340             :         }
    6341             :         /* project (join (A,B)[ A.x = B.y ] ) [project_cols] -> project (A) [project_cols]
    6342             :          * where non of the project_cols are from B and x=y is a foreign key join (B is the unique side)
    6343             :          * and there are no filters on B
    6344             :          */
    6345             :         if (is_project(rel->op)) {
    6346             :                 sql_rel *j = rel->l;
    6347             : 
    6348             :                 if (is_join(j->op)) {
    6349             :                         node *n;
    6350             :                         sql_rel *l = j->l;
    6351             :                         sql_rel *r = j->r;
    6352             : 
    6353             :                         if (!l || rel_is_ref(l) || !r || rel_is_ref(r) || r->op != op_basetable || r->l)
    6354             :                                 return rel;
    6355             : 
    6356             :                         /* check if all projection cols can be found in l */
    6357             :                         for(n = rel->exps->h; n; n = n->next) {
    6358             :                                 sql_exp *e = n->data;
    6359             : 
    6360             :                                 if (!rel_find_exp(l, e))
    6361             :                                         return rel;
    6362             : 
    6363             :                         }
    6364             :                         assert(0);
    6365             :                         v->changes++;
    6366             :                         rel->l = l;
    6367             :                         rel->r = NULL;
    6368             :                 }
    6369             :         }
    6370             :         return rel;
    6371             : }
    6372             : 
    6373             : /* Pushing projects up the tree. Done very early in the optimizer.
    6374             :  * Makes later steps easier.
    6375             :  */
    6376             : static sql_rel *
    6377     4979851 : rel_push_project_up(visitor *v, sql_rel *rel)
    6378             : {
    6379     4979851 :         if (is_simple_project(rel->op) && rel->l && !rel_is_ref(rel)) {
    6380             :                 sql_rel *l = rel->l;
    6381     1348377 :                 if (is_simple_project(l->op))
    6382      401215 :                         return rel_merge_projects(v, rel);
    6383             :         }
    6384             : 
    6385             :         /* project/project cleanup is done later */
    6386     4578636 :         if (is_join(rel->op) || is_select(rel->op)) {
    6387             :                 node *n;
    6388             :                 list *exps = NULL, *l_exps, *r_exps;
    6389     1557210 :                 sql_rel *l = rel->l;
    6390     1557210 :                 sql_rel *r = rel->r;
    6391             :                 sql_rel *t;
    6392             :                 int nlexps = 0, i = 0;
    6393             : 
    6394             :                 /* Don't rewrite refs, non projections or constant or
    6395             :                    order by projections  */
    6396     1557210 :                 if (!l || rel_is_ref(l) || is_topn(l->op) || is_sample(l->op) ||
    6397     1417573 :                    (is_join(rel->op) && (!r || rel_is_ref(r))) ||
    6398     1391128 :                    (is_left(rel->op) && (rel->flag&MERGE_LEFT) /* can't push projections above merge statments left joins */) ||
    6399     1390939 :                    (is_select(rel->op) && l->op != op_project) ||
    6400      860819 :                    (is_join(rel->op) && ((l->op != op_project && r->op != op_project) || is_topn(r->op) || is_sample(r->op))) ||
    6401      195833 :                   ((l->op == op_project && (!l->l || l->r || project_unsafe(l,is_select(rel->op)))) ||
    6402      107926 :                    (is_join(rel->op) && (r->op == op_project && (!r->l || r->r || project_unsafe(r,0))))))
    6403     1492093 :                         return rel;
    6404             : 
    6405       65117 :                 if (l->op == op_project && l->l) {
    6406             :                         /* Go through the list of project expressions.
    6407             :                            Check if they can be pushed up, ie are they not
    6408             :                            changing or introducing any columns used
    6409             :                            by the upper operator. */
    6410             : 
    6411       49256 :                         exps = new_exp_list(v->sql->sa);
    6412     1049135 :                         for (n = l->exps->h; n; n = n->next) {
    6413     1007270 :                                 sql_exp *e = n->data;
    6414             : 
    6415             :                                 /* we cannot rewrite projection with atomic values from outer joins */
    6416     1007270 :                                 if (is_column(e->type) && exp_is_atom(e) && !(is_right(rel->op) || is_full(rel->op))) {
    6417         383 :                                         list_append(exps, e);
    6418     1006887 :                                 } else if (e->type == e_column) {
    6419     1002152 :                                         if (has_label(e))
    6420             :                                                 return rel;
    6421      999496 :                                         list_append(exps, e);
    6422             :                                 } else {
    6423             :                                         return rel;
    6424             :                                 }
    6425             :                         }
    6426             :                 } else {
    6427       15861 :                         exps = rel_projections(v->sql, l, NULL, 1, 1);
    6428             :                 }
    6429       57726 :                 nlexps = list_length(exps);
    6430             :                 /* also handle right hand of join */
    6431       57726 :                 if (is_join(rel->op) && r->op == op_project && r->l) {
    6432             :                         /* Here we also check all expressions of r like above
    6433             :                            but also we need to check for ambigious names. */
    6434             : 
    6435       65972 :                         for (n = r->exps->h; n; n = n->next) {
    6436       59872 :                                 sql_exp *e = n->data;
    6437             : 
    6438             :                                 /* we cannot rewrite projection with atomic values from outer joins */
    6439       59872 :                                 if (is_column(e->type) && exp_is_atom(e) && !(is_left(rel->op) || is_full(rel->op))) {
    6440         144 :                                         list_append(exps, e);
    6441       59728 :                                 } else if (e->type == e_column) {
    6442       59375 :                                         if (has_label(e))
    6443             :                                                 return rel;
    6444       47736 :                                         list_append(exps, e);
    6445             :                                 } else {
    6446             :                                         return rel;
    6447             :                                 }
    6448             :                         }
    6449       39634 :                 } else if (is_join(rel->op)) {
    6450       22770 :                         list *r_exps = rel_projections(v->sql, r, NULL, 1, 1);
    6451       22770 :                         list_merge(exps, r_exps, (fdup)NULL);
    6452             :                 }
    6453             :                 /* Here we should check for ambigious names ? */
    6454       45734 :                 if (is_join(rel->op) && r) {
    6455       28870 :                         t = (l->op == op_project && l->l)?l->l:l;
    6456       28870 :                         l_exps = rel_projections(v->sql, t, NULL, 1, 1);
    6457             :                         /* conflict with old right expressions */
    6458       28870 :                         r_exps = rel_projections(v->sql, r, NULL, 1, 1);
    6459      752459 :                         for(n = l_exps->h; n; n = n->next) {
    6460      723915 :                                 sql_exp *e = n->data;
    6461      723915 :                                 const char *rname = exp_relname(e);
    6462      723915 :                                 const char *name = exp_name(e);
    6463             : 
    6464      723915 :                                 if (exp_is_atom(e))
    6465           0 :                                         continue;
    6466      723915 :                                 if ((rname && exps_bind_column2(r_exps, rname, name, NULL) != NULL) ||
    6467         237 :                                     (!rname && exps_bind_column(r_exps, name, NULL, NULL, 1) != NULL))
    6468         326 :                                         return rel;
    6469             :                         }
    6470       28544 :                         t = (r->op == op_project && r->l)?r->l:r;
    6471       28544 :                         r_exps = rel_projections(v->sql, t, NULL, 1, 1);
    6472             :                         /* conflict with new right expressions */
    6473      738032 :                         for(n = l_exps->h; n; n = n->next) {
    6474      711121 :                                 sql_exp *e = n->data;
    6475             : 
    6476      711121 :                                 if (exp_is_atom(e))
    6477           0 :                                         continue;
    6478     1420630 :                                 if ((e->l && exps_bind_column2(r_exps, e->l, e->r, NULL) != NULL) ||
    6479      709530 :                                    (exps_bind_column(r_exps, e->r, NULL, NULL, 1) != NULL && (!e->l || !e->r)))
    6480        1633 :                                         return rel;
    6481             :                         }
    6482             :                         /* conflict with new left expressions */
    6483      178871 :                         for(n = r_exps->h; n; n = n->next) {
    6484      151960 :                                 sql_exp *e = n->data;
    6485             : 
    6486      151960 :                                 if (exp_is_atom(e))
    6487           0 :                                         continue;
    6488      303920 :                                 if ((e->l && exps_bind_column2(l_exps, e->l, e->r, NULL) != NULL) ||
    6489      151964 :                                    (exps_bind_column(l_exps, e->r, NULL, NULL, 1) != NULL && (!e->l || !e->r)))
    6490           0 :                                         return rel;
    6491             :                         }
    6492             :                 }
    6493             : 
    6494             :                 /* rename operator expressions */
    6495       43775 :                 if (l->op == op_project) {
    6496             :                         /* rewrite rel from rel->l into rel->l->l */
    6497       39508 :                         if (rel->exps) {
    6498       84727 :                                 for (n = rel->exps->h; n; n = n->next) {
    6499       46462 :                                         sql_exp *e = n->data, *ne;
    6500             : 
    6501       46462 :                                         ne = exp_rename(v->sql, e, l, l->l);
    6502       46462 :                                         assert(ne);
    6503       46462 :                                         if (ne != e && exp_name(e))
    6504           0 :                                                 exp_propagate(v->sql->sa, ne, e);
    6505       46462 :                                         n->data = ne;
    6506             :                                 }
    6507             :                         }
    6508       39508 :                         rel->l = l->l;
    6509       39508 :                         l->l = NULL;
    6510       39508 :                         rel_destroy(l);
    6511             :                 }
    6512       43775 :                 if (is_join(rel->op) && r->op == op_project) {
    6513             :                         /* rewrite rel from rel->r into rel->r->l */
    6514        4463 :                         if (rel->exps) {
    6515        8580 :                                 for (n = rel->exps->h; n; n = n->next) {
    6516        4525 :                                         sql_exp *e = n->data, *ne;
    6517             : 
    6518        4525 :                                         ne = exp_rename(v->sql, e, r, r->l);
    6519        4525 :                                         assert(ne);
    6520        4525 :                                         if (ne != e && exp_name(e))
    6521           0 :                                                 exp_propagate(v->sql->sa, ne, e);
    6522        4525 :                                         n->data = ne;
    6523             :                                 }
    6524             :                         }
    6525        4463 :                         rel->r = r->l;
    6526        4463 :                         r->l = NULL;
    6527        4463 :                         rel_destroy(r);
    6528             :                 }
    6529             :                 /* Done, ie introduce new project */
    6530       43775 :                 exps_fix_card(exps, rel->card);
    6531             :                 /* Fix nil flag */
    6532       43775 :                 if (!list_empty(exps)) {
    6533     1021861 :                         for (n = exps->h ; n && i < nlexps ; n = n->next, i++) {
    6534      978086 :                                 sql_exp *e = n->data;
    6535             : 
    6536      978086 :                                 if (is_right(rel->op) || is_full(rel->op))
    6537         369 :                                         set_has_nil(e);
    6538      978086 :                                 set_not_unique(e);
    6539             :                         }
    6540      186911 :                         for (; n ; n = n->next) {
    6541      143136 :                                 sql_exp *e = n->data;
    6542             : 
    6543      143136 :                                 if (is_left(rel->op) || is_full(rel->op))
    6544       53410 :                                         set_has_nil(e);
    6545      143136 :                                 set_not_unique(e);
    6546             :                         }
    6547             :                 }
    6548       43775 :                 v->changes++;
    6549       43775 :                 return rel_inplace_project(v->sql->sa, rel, NULL, exps);
    6550             :         }
    6551     3021426 :         if (is_groupby(rel->op) && !rel_is_ref(rel) && rel->exps && list_length(rel->exps) > 1) {
    6552             :                 node *n;
    6553             :                 int fnd = 0;
    6554             :                 list *aexps, *pexps;
    6555             : 
    6556             :                 /* check if some are expressions aren't e_aggr */
    6557      356022 :                 for (n = rel->exps->h; n && !fnd; n = n->next) {
    6558      263469 :                         sql_exp *e = n->data;
    6559             : 
    6560      263469 :                         if (e->type != e_aggr && e->type != e_column && e->type != e_atom) {
    6561             :                                 fnd = 1;
    6562             :                         }
    6563             :                 }
    6564             :                 /* only aggr, no rewrite needed */
    6565       92553 :                 if (!fnd)
    6566             :                         return rel;
    6567             : 
    6568           5 :                 aexps = sa_list(v->sql->sa);
    6569           5 :                 pexps = sa_list(v->sql->sa);
    6570          30 :                 for (n = rel->exps->h; n; n = n->next) {
    6571          25 :                         sql_exp *e = n->data, *ne = NULL;
    6572             : 
    6573          25 :                         switch (e->type) {
    6574           0 :                         case e_atom: /* move over to the projection */
    6575           0 :                                 list_append(pexps, e);
    6576           0 :                                 break;
    6577           5 :                         case e_func:
    6578           5 :                                 list_append(pexps, e);
    6579           5 :                                 list_split_aggr_and_project(v->sql, aexps, e->l);
    6580           5 :                                 break;
    6581           0 :                         case e_convert:
    6582           0 :                                 list_append(pexps, e);
    6583           0 :                                 e->l = split_aggr_and_project(v->sql, aexps, e->l);
    6584           0 :                                 break;
    6585          20 :                         default: /* simple alias */
    6586          20 :                                 list_append(aexps, e);
    6587          20 :                                 ne = exp_column(v->sql->sa, exp_find_rel_name(e), exp_name(e), exp_subtype(e), e->card, has_nil(e), is_unique(e), is_intern(e));
    6588          20 :                                 list_append(pexps, ne);
    6589          20 :                                 break;
    6590             :                         }
    6591             :                 }
    6592           5 :                 v->changes++;
    6593           5 :                 rel->exps = aexps;
    6594           5 :                 return rel_inplace_project( v->sql->sa, rel, NULL, pexps);
    6595             :         }
    6596             :         return rel;
    6597             : }
    6598             : 
    6599             : /* if local_proj is set: the current expression is from the same projection */
    6600             : static int exp_mark_used(sql_rel *subrel, sql_exp *e, int local_proj);
    6601             : 
    6602             : static int
    6603      471838 : exps_mark_used(sql_rel *subrel, list *l, int local_proj)
    6604             : {
    6605             :         int nr = 0;
    6606      471838 :         if (list_empty(l))
    6607             :                 return nr;
    6608             : 
    6609     1482323 :         for (node *n = l->h; n != NULL; n = n->next)
    6610     1010495 :                 nr += exp_mark_used(subrel, n->data, local_proj);
    6611             :         return nr;
    6612             : }
    6613             : 
    6614             : static int
    6615     4750603 : exp_mark_used(sql_rel *subrel, sql_exp *e, int local_proj)
    6616             : {
    6617             :         int nr = 0;
    6618             :         sql_exp *ne = NULL;
    6619             : 
    6620     5054498 :         switch(e->type) {
    6621     3343135 :         case e_column:
    6622     3343135 :                 ne = rel_find_exp(subrel, e);
    6623     3343135 :                 break;
    6624      303895 :         case e_convert:
    6625      303895 :                 return exp_mark_used(subrel, e->l, local_proj);
    6626      430873 :         case e_aggr:
    6627             :         case e_func: {
    6628      430873 :                 if (e->l)
    6629      418056 :                         nr += exps_mark_used(subrel, e->l, local_proj);
    6630      430873 :                 assert(!e->r);
    6631             :                 break;
    6632             :         }
    6633      422569 :         case e_cmp:
    6634      422569 :                 if (e->flag == cmp_or || e->flag == cmp_filter) {
    6635       15708 :                         nr += exps_mark_used(subrel, e->l, local_proj);
    6636       15708 :                         nr += exps_mark_used(subrel, e->r, local_proj);
    6637      406861 :                 } else if (e->flag == cmp_in || e->flag == cmp_notin) {
    6638       17622 :                         nr += exp_mark_used(subrel, e->l, local_proj);
    6639       17622 :                         nr += exps_mark_used(subrel, e->r, local_proj);
    6640             :                 } else {
    6641      389239 :                         nr += exp_mark_used(subrel, e->l, local_proj);
    6642      389239 :                         nr += exp_mark_used(subrel, e->r, local_proj);
    6643      389239 :                         if (e->f)
    6644        6368 :                                 nr += exp_mark_used(subrel, e->f, local_proj);
    6645             :                 }
    6646             :                 break;
    6647      554026 :         case e_atom:
    6648             :                 /* atoms are used in e_cmp */
    6649      554026 :                 e->used = 1;
    6650             :                 /* return 0 as constants may require a full column ! */
    6651      554026 :                 if (e->f)
    6652        4744 :                         nr += exps_mark_used(subrel, e->f, local_proj);
    6653             :                 return nr;
    6654           0 :         case e_psm:
    6655           0 :                 if (e->flag & PSM_SET || e->flag & PSM_RETURN || e->flag & PSM_EXCEPTION) {
    6656           0 :                         nr += exp_mark_used(subrel, e->l, local_proj);
    6657           0 :                 } else if (e->flag & PSM_WHILE || e->flag & PSM_IF) {
    6658           0 :                         nr += exp_mark_used(subrel, e->l, local_proj);
    6659           0 :                         nr += exps_mark_used(subrel, e->r, local_proj);
    6660           0 :                         if (e->flag == PSM_IF && e->f)
    6661           0 :                                 nr += exps_mark_used(subrel, e->f, local_proj);
    6662             :                 }
    6663           0 :                 e->used = 1;
    6664           0 :                 break;
    6665             :         }
    6666     4196577 :         if (ne && e != ne) {
    6667     1867766 :                 if (!local_proj || ne->type != e_column || (has_label(ne) || (ne->alias.rname && ne->alias.rname[0] == '%')) || (subrel->l && !rel_find_exp(subrel->l, e)))
    6668     1761338 :                         ne->used = 1;
    6669     1867766 :                 return ne->used;
    6670             :         }
    6671             :         return nr;
    6672             : }
    6673             : 
    6674             : static void
    6675       34712 : positional_exps_mark_used( sql_rel *rel, sql_rel *subrel )
    6676             : {
    6677       34712 :         assert(rel->exps);
    6678             : 
    6679       34712 :         if ((is_topn(subrel->op) || is_sample(subrel->op)) && subrel->l)
    6680             :                 subrel = subrel->l;
    6681             :         /* everything is used within the set operation */
    6682       34712 :         if (rel->exps && subrel->exps) {
    6683             :                 node *m;
    6684      268398 :                 for (m=subrel->exps->h; m; m = m->next) {
    6685      233686 :                         sql_exp *se = m->data;
    6686             : 
    6687      233686 :                         se->used = 1;
    6688             :                 }
    6689             :         }
    6690       34712 : }
    6691             : 
    6692             : static void
    6693      627091 : rel_exps_mark_used(sql_allocator *sa, sql_rel *rel, sql_rel *subrel)
    6694             : {
    6695             :         int nr = 0;
    6696             : 
    6697      627091 :         if (rel->r && (is_simple_project(rel->op) || is_groupby(rel->op))) {
    6698             :                 list *l = rel->r;
    6699             :                 node *n;
    6700             : 
    6701      104992 :                 for (n=l->h; n; n = n->next) {
    6702       73474 :                         sql_exp *e = n->data;
    6703             : 
    6704       73474 :                         e->used = 1;
    6705       73474 :                         exp_mark_used(rel, e, 1);
    6706             :                 }
    6707             :         }
    6708             : 
    6709      627091 :         if (rel->exps) {
    6710             :                 node *n;
    6711      595501 :                 int len = list_length(rel->exps), i;
    6712      595501 :                 sql_exp **exps = SA_NEW_ARRAY(sa, sql_exp*, len);
    6713             : 
    6714     2425817 :                 for (n=rel->exps->h, i = 0; n; n = n->next, i++) {
    6715     1830316 :                         sql_exp *e = exps[i] = n->data;
    6716             : 
    6717     1830316 :                         nr += e->used;
    6718             :                 }
    6719             : 
    6720      595501 :                 if (!nr && is_project(rel->op) && len > 0) /* project at least one column if exists */
    6721        3632 :                         exps[0]->used = 1;
    6722             : 
    6723     2425817 :                 for (i = len-1; i >= 0; i--) {
    6724     1830316 :                         sql_exp *e = exps[i];
    6725             : 
    6726     1830316 :                         if (!is_project(rel->op) || e->used) {
    6727     1592421 :                                 if (is_project(rel->op))
    6728     1197626 :                                         nr += exp_mark_used(rel, e, 1);
    6729     1592421 :                                 nr += exp_mark_used(subrel, e, 0);
    6730             :                         }
    6731             :                 }
    6732             :         }
    6733             :         /* for count/rank we need atleast one column */
    6734      627091 :         if (!nr && subrel && (is_project(subrel->op) || is_base(subrel->op)) && !list_empty(subrel->exps) &&
    6735       12672 :                 (is_simple_project(rel->op) && project_unsafe(rel, 0))) {
    6736          21 :                 sql_exp *e = subrel->exps->h->data;
    6737          21 :                 e->used = 1;
    6738             :         }
    6739      627091 :         if (rel->r && (is_simple_project(rel->op) || is_groupby(rel->op))) {
    6740             :                 list *l = rel->r;
    6741             :                 node *n;
    6742             : 
    6743      104992 :                 for (n=l->h; n; n = n->next) {
    6744       73474 :                         sql_exp *e = n->data;
    6745             : 
    6746       73474 :                         e->used = 1;
    6747             :                         /* possibly project/groupby uses columns from the inner */
    6748       73474 :                         exp_mark_used(subrel, e, 0);
    6749             :                 }
    6750             :         }
    6751      627091 : }
    6752             : 
    6753             : static void exps_used(list *l);
    6754             : 
    6755             : static void
    6756     2448558 : exp_used(sql_exp *e)
    6757             : {
    6758     2490179 :         if (e) {
    6759     2480744 :                 e->used = 1;
    6760             : 
    6761     2480744 :                 switch (e->type) {
    6762       40426 :                 case e_convert:
    6763       40426 :                         exp_used(e->l);
    6764       40426 :                         break;
    6765      111550 :                 case e_func:
    6766             :                 case e_aggr:
    6767      111550 :                         exps_used(e->l);
    6768      111550 :                         break;
    6769        8551 :                 case e_cmp:
    6770        8551 :                         if (e->flag == cmp_or || e->flag == cmp_filter) {
    6771         883 :                                 exps_used(e->l);
    6772         883 :                                 exps_used(e->r);
    6773        7668 :                         } else if (e->flag == cmp_in || e->flag == cmp_notin) {
    6774           3 :                                 exp_used(e->l);
    6775           3 :                                 exps_used(e->r);
    6776             :                         } else {
    6777        7665 :                                 exp_used(e->l);
    6778        7665 :                                 exp_used(e->r);
    6779        7665 :                                 if (e->f)
    6780             :                                         exp_used(e->f);
    6781             :                         }
    6782             :                         break;
    6783             :                 default:
    6784             :                         break;
    6785             :                 }
    6786        9435 :         }
    6787     2448558 : }
    6788             : 
    6789             : static void
    6790      652093 : exps_used(list *l)
    6791             : {
    6792      652093 :         if (l) {
    6793     3081133 :                 for (node *n = l->h; n; n = n->next)
    6794     2431159 :                         exp_used(n->data);
    6795             :         }
    6796      652093 : }
    6797             : 
    6798             : static void
    6799      508557 : rel_used(sql_rel *rel)
    6800             : {
    6801      508557 :         if (!rel)
    6802             :                 return;
    6803      506590 :         if (is_join(rel->op) || is_set(rel->op) || is_semi(rel->op) || is_modify(rel->op)) {
    6804       31349 :                 rel_used(rel->l);
    6805       31349 :                 rel_used(rel->r);
    6806      475241 :         } else if (is_topn(rel->op) || is_select(rel->op) || is_sample(rel->op)) {
    6807        4036 :                 rel_used(rel->l);
    6808        4036 :                 rel = rel->l;
    6809      471205 :         } else if (is_ddl(rel->op)) {
    6810      311137 :                 if (rel->flag == ddl_output || rel->flag == ddl_create_seq || rel->flag == ddl_alter_seq || rel->flag == ddl_alter_table || rel->flag == ddl_create_table || rel->flag == ddl_create_view) {
    6811       26566 :                         rel_used(rel->l);
    6812      284571 :                 } else if (rel->flag == ddl_list || rel->flag == ddl_exception) {
    6813         489 :                         rel_used(rel->l);
    6814         489 :                         rel_used(rel->r);
    6815      284082 :                 } else if (rel->flag == ddl_psm) {
    6816       19105 :                         exps_used(rel->exps);
    6817             :                 }
    6818      160068 :         } else if (rel->op == op_table) {
    6819        2066 :                 if (IS_TABLE_PROD_FUNC(rel->flag) || rel->flag == TABLE_FROM_RELATION)
    6820        2066 :                         rel_used(rel->l);
    6821        2066 :                 exp_used(rel->r);
    6822             :         }
    6823      506590 :         if (rel && rel->exps) {
    6824      499560 :                 exps_used(rel->exps);
    6825      499560 :                 if (rel->r && (is_simple_project(rel->op) || is_groupby(rel->op)))
    6826       20109 :                         exps_used(rel->r);
    6827             :         }
    6828             : }
    6829             : 
    6830             : static void
    6831      946152 : rel_mark_used(mvc *sql, sql_rel *rel, int proj)
    6832             : {
    6833     1436249 :         if (proj && (need_distinct(rel)))
    6834        9206 :                 rel_used(rel);
    6835             : 
    6836     1436249 :         switch(rel->op) {
    6837             :         case op_basetable:
    6838             :         case op_truncate:
    6839             :         case op_insert:
    6840             :                 break;
    6841             : 
    6842       17726 :         case op_table:
    6843             : 
    6844       17726 :                 if (rel->l && rel->flag != TRIGGER_WRAPPER) {
    6845         646 :                         rel_used(rel);
    6846         646 :                         if (rel->r)
    6847         646 :                                 exp_mark_used(rel->l, rel->r, 0);
    6848         646 :                         rel_mark_used(sql, rel->l, proj);
    6849             :                 }
    6850             :                 break;
    6851             : 
    6852         485 :         case op_topn:
    6853             :         case op_sample:
    6854         485 :                 if (proj) {
    6855         437 :                         rel = rel ->l;
    6856             :                         rel_mark_used(sql, rel, proj);
    6857         437 :                         break;
    6858             :                 }
    6859             :                 /* fall through */
    6860             :         case op_project:
    6861             :         case op_groupby:
    6862      339108 :                 if (proj && rel->l) {
    6863      194073 :                         rel_exps_mark_used(sql->sa, rel, rel->l);
    6864      194074 :                         rel_mark_used(sql, rel->l, 0);
    6865      145035 :                 } else if (proj) {
    6866        5838 :                         rel_exps_mark_used(sql->sa, rel, NULL);
    6867             :                 }
    6868             :                 break;
    6869        6317 :         case op_update:
    6870             :         case op_delete:
    6871        6317 :                 if (proj && rel->r) {
    6872             :                         sql_rel *r = rel->r;
    6873             : 
    6874        5234 :                         if (!list_empty(r->exps)) {
    6875        6130 :                                 for (node *n = r->exps->h; n; n = n->next) {
    6876        5235 :                                         sql_exp *e = n->data;
    6877        5235 :                                         const char *nname = exp_name(e);
    6878             : 
    6879        5235 :                                         if (nname[0] == '%' && strcmp(nname, TID) == 0) { /* TID is used */
    6880        4339 :                                                 e->used = 1;
    6881        4339 :                                                 break;
    6882             :                                         }
    6883             :                                 }
    6884             :                         }
    6885        5234 :                         rel_exps_mark_used(sql->sa, rel, rel->r);
    6886        5234 :                         rel_mark_used(sql, rel->r, 0);
    6887             :                 }
    6888             :                 break;
    6889             : 
    6890      310182 :         case op_ddl:
    6891      310182 :                 if (rel->flag == ddl_output || rel->flag == ddl_create_seq || rel->flag == ddl_alter_seq || rel->flag == ddl_alter_table || rel->flag == ddl_create_table || rel->flag == ddl_create_view) {
    6892       25611 :                         if (rel->l)
    6893             :                                 rel_mark_used(sql, rel->l, 0);
    6894      284571 :                 } else if (rel->flag == ddl_list || rel->flag == ddl_exception) {
    6895         489 :                         if (rel->l)
    6896         482 :                                 rel_mark_used(sql, rel->l, 0);
    6897         489 :                         if (rel->r)
    6898             :                                 rel_mark_used(sql, rel->r, 0);
    6899             :                 }
    6900             :                 break;
    6901             : 
    6902       97396 :         case op_select:
    6903       97396 :                 if (rel->l) {
    6904       97396 :                         rel_exps_mark_used(sql->sa, rel, rel->l);
    6905       97396 :                         rel_mark_used(sql, rel->l, 0);
    6906             :                 }
    6907             :                 break;
    6908             : 
    6909       43089 :         case op_union:
    6910             :         case op_inter:
    6911             :         case op_except:
    6912             :                 /* For now we mark all union expression as used */
    6913             : 
    6914             :                 /* Later we should (in case of union all) remove unused
    6915             :                  * columns from the projection.
    6916             :                  *
    6917             :                  * Project part of union is based on column position.
    6918             :                  */
    6919       43089 :                 if (proj && (need_distinct(rel) || !rel->exps)) {
    6920        4304 :                         rel_used(rel);
    6921        4304 :                         if (!rel->exps) {
    6922           0 :                                 rel_used(rel->l);
    6923           0 :                                 rel_used(rel->r);
    6924             :                         }
    6925        4304 :                         rel_mark_used(sql, rel->l, 0);
    6926        4304 :                         rel_mark_used(sql, rel->r, 0);
    6927       38785 :                 } else if (proj && !need_distinct(rel)) {
    6928       17356 :                         sql_rel *l = rel->l;
    6929             : 
    6930       17356 :                         positional_exps_mark_used(rel, l);
    6931       17356 :                         rel_exps_mark_used(sql->sa, rel, l);
    6932       17356 :                         rel_mark_used(sql, rel->l, 0);
    6933             :                         /* based on child check set expression list */
    6934       17356 :                         if (is_project(l->op) && need_distinct(l))
    6935           0 :                                 positional_exps_mark_used(l, rel);
    6936       17356 :                         positional_exps_mark_used(rel, rel->r);
    6937       17356 :                         rel_exps_mark_used(sql->sa, rel, rel->r);
    6938       17355 :                         rel_mark_used(sql, rel->r, 0);
    6939             :                 }
    6940             :                 break;
    6941             : 
    6942      144919 :         case op_join:
    6943             :         case op_left:
    6944             :         case op_right:
    6945             :         case op_full:
    6946             :         case op_semi:
    6947             :         case op_anti:
    6948             :         case op_merge:
    6949      144919 :                 rel_exps_mark_used(sql->sa, rel, rel->l);
    6950      144919 :                 rel_exps_mark_used(sql->sa, rel, rel->r);
    6951      144919 :                 rel_mark_used(sql, rel->l, 0);
    6952      144919 :                 rel_mark_used(sql, rel->r, 0);
    6953      144919 :                 break;
    6954             :         }
    6955      946152 : }
    6956             : 
    6957             : static sql_rel * rel_dce_sub(mvc *sql, sql_rel *rel);
    6958             : 
    6959             : static sql_rel *
    6960      813813 : rel_remove_unused(mvc *sql, sql_rel *rel)
    6961             : {
    6962             :         int needed = 0;
    6963             : 
    6964      813813 :         if (!rel)
    6965             :                 return rel;
    6966             : 
    6967      805336 :         switch(rel->op) {
    6968      256046 :         case op_basetable: {
    6969      256046 :                 sql_table *t = rel->l;
    6970             : 
    6971      256046 :                 if (t && isReplicaTable(t)) /* TODO fix rewriting in rel_distribute.c */
    6972             :                         return rel;
    6973             :         }
    6974             :         /* fall through */
    6975             :         case op_table:
    6976      264827 :                 if (rel->exps && (rel->op != op_table || !IS_TABLE_PROD_FUNC(rel->flag))) {
    6977     1104598 :                         for(node *n=rel->exps->h; n && !needed; n = n->next) {
    6978      848248 :                                 sql_exp *e = n->data;
    6979             : 
    6980      848248 :                                 if (!e->used)
    6981             :                                         needed = 1;
    6982             :                         }
    6983             : 
    6984      256350 :                         if (!needed)
    6985             :                                 return rel;
    6986             : 
    6987     1230971 :                         for(node *n=rel->exps->h; n;) {
    6988     1027595 :                                 node *next = n->next;
    6989     1027595 :                                 sql_exp *e = n->data;
    6990             : 
    6991             :                                 /* atleast one (needed for crossproducts, count(*), rank() and single value projections) !, handled by rel_exps_mark_used */
    6992     1027595 :                                 if (!e->used && list_length(rel->exps) > 1)
    6993      395026 :                                         list_remove_node(rel->exps, NULL, n);
    6994             :                                 n = next;
    6995             :                         }
    6996             :                 }
    6997      211853 :                 if (rel->op == op_table && (IS_TABLE_PROD_FUNC(rel->flag) || rel->flag == TABLE_FROM_RELATION))
    6998        8477 :                         rel->l = rel_remove_unused(sql, rel->l);
    6999             :                 return rel;
    7000             : 
    7001         437 :         case op_topn:
    7002             :         case op_sample:
    7003             : 
    7004         437 :                 if (rel->l)
    7005         437 :                         rel->l = rel_remove_unused(sql, rel->l);
    7006             :                 return rel;
    7007             : 
    7008      199873 :         case op_project:
    7009             :         case op_groupby:
    7010             : 
    7011      199873 :                 if (/*rel->l &&*/ rel->exps) {
    7012     1186737 :                         for(node *n=rel->exps->h; n && !needed; n = n->next) {
    7013      986864 :                                 sql_exp *e = n->data;
    7014             : 
    7015      986864 :                                 if (!e->used)
    7016             :                                         needed = 1;
    7017             :                         }
    7018      199873 :                         if (!needed)
    7019             :                                 return rel;
    7020             : 
    7021      282229 :                         for(node *n=rel->exps->h; n;) {
    7022      261934 :                                 node *next = n->next;
    7023      261934 :                                 sql_exp *e = n->data;
    7024             : 
    7025             :                                 /* atleast one (needed for crossproducts, count(*), rank() and single value projections) */
    7026      261934 :                                 if (!e->used && list_length(rel->exps) > 1)
    7027      171161 :                                         list_remove_node(rel->exps, NULL, n);
    7028             :                                 n = next;
    7029             :                         }
    7030             :                 }
    7031             :                 return rel;
    7032             : 
    7033             :         case op_union:
    7034             :         case op_inter:
    7035             :         case op_except:
    7036             : 
    7037             :         case op_insert:
    7038             :         case op_update:
    7039             :         case op_delete:
    7040             :         case op_truncate:
    7041             :         case op_merge:
    7042             : 
    7043             :         case op_select:
    7044             : 
    7045             :         case op_join:
    7046             :         case op_left:
    7047             :         case op_right:
    7048             :         case op_full:
    7049             :         case op_semi:
    7050             :         case op_anti:
    7051             :                 return rel;
    7052      309784 :         case op_ddl:
    7053      309784 :                 if (rel->flag == ddl_output || rel->flag == ddl_create_seq || rel->flag == ddl_alter_seq || rel->flag == ddl_alter_table || rel->flag == ddl_create_table || rel->flag == ddl_create_view) {
    7054       25252 :                         if (rel->l)
    7055       24915 :                                 rel->l = rel_remove_unused(sql, rel->l);
    7056      284532 :                 } else if (rel->flag == ddl_list || rel->flag == ddl_exception) {
    7057         450 :                         if (rel->l)
    7058         443 :                                 rel->l = rel_remove_unused(sql, rel->l);
    7059         450 :                         if (rel->r)
    7060         450 :                                 rel->r = rel_remove_unused(sql, rel->r);
    7061             :                 }
    7062             :                 return rel;
    7063             :         }
    7064             :         return rel;
    7065             : }
    7066             : 
    7067             : static void
    7068     1070177 : rel_dce_refs(mvc *sql, sql_rel *rel, list *refs)
    7069             : {
    7070     1070177 :         if (!rel || (rel_is_ref(rel) && list_find(refs, rel, NULL)))
    7071       11867 :                 return ;
    7072             : 
    7073     1058310 :         switch(rel->op) {
    7074      322874 :         case op_table:
    7075             :         case op_topn:
    7076             :         case op_sample:
    7077             :         case op_project:
    7078             :         case op_groupby:
    7079             :         case op_select:
    7080             : 
    7081      322874 :                 if (rel->l && (rel->op != op_table || rel->flag != TRIGGER_WRAPPER))
    7082      308442 :                         rel_dce_refs(sql, rel->l, refs);
    7083             :                 break;
    7084             : 
    7085             :         case op_basetable:
    7086             :         case op_insert:
    7087             :         case op_truncate:
    7088             :                 break;
    7089             : 
    7090        5337 :         case op_update:
    7091             :         case op_delete:
    7092             : 
    7093        5337 :                 if (rel->r)
    7094        5234 :                         rel_dce_refs(sql, rel->r, refs);
    7095             :                 break;
    7096             : 
    7097      172537 :         case op_union:
    7098             :         case op_inter:
    7099             :         case op_except:
    7100             :         case op_join:
    7101             :         case op_left:
    7102             :         case op_right:
    7103             :         case op_full:
    7104             :         case op_semi:
    7105             :         case op_anti:
    7106             :         case op_merge:
    7107             : 
    7108      172537 :                 if (rel->l)
    7109      172537 :                         rel_dce_refs(sql, rel->l, refs);
    7110      172537 :                 if (rel->r)
    7111      172537 :                         rel_dce_refs(sql, rel->r, refs);
    7112             :                 break;
    7113      311077 :         case op_ddl:
    7114             : 
    7115      311077 :                 if (rel->flag == ddl_output || rel->flag == ddl_create_seq || rel->flag == ddl_alter_seq || rel->flag == ddl_alter_table || rel->flag == ddl_create_table || rel->flag == ddl_create_view) {
    7116       26506 :                         if (rel->l)
    7117       26169 :                                 rel_dce_refs(sql, rel->l, refs);
    7118      284571 :                 } else if (rel->flag == ddl_list || rel->flag == ddl_exception) {
    7119         489 :                         if (rel->l)
    7120         482 :                                 rel_dce_refs(sql, rel->l, refs);
    7121         489 :                         if (rel->r)
    7122         458 :                                 rel_dce_refs(sql, rel->r, refs);
    7123             :                 } break;
    7124             :         }
    7125             : 
    7126     1058310 :         if (rel_is_ref(rel) && !list_find(refs, rel, NULL))
    7127       10422 :                 list_prepend(refs, rel);
    7128             : }
    7129             : 
    7130             : static sql_rel *
    7131     1422777 : rel_dce_down(mvc *sql, sql_rel *rel, int skip_proj)
    7132             : {
    7133     1422777 :         if (!rel)
    7134             :                 return rel;
    7135             : 
    7136     1422777 :         if (!skip_proj && rel_is_ref(rel))
    7137             :                 return rel;
    7138             : 
    7139     1404557 :         switch(rel->op) {
    7140      479445 :         case op_basetable:
    7141             :         case op_table:
    7142             : 
    7143      479445 :                 if (skip_proj && rel->l && rel->op == op_table && rel->flag != TRIGGER_WRAPPER)
    7144         323 :                         rel->l = rel_dce_down(sql, rel->l, 0);
    7145      479445 :                 if (!skip_proj)
    7146      239502 :                         rel_dce_sub(sql, rel);
    7147             :                 /* fall through */
    7148             : 
    7149             :         case op_truncate:
    7150             :                 return rel;
    7151             : 
    7152        3317 :         case op_insert:
    7153        3317 :                 rel_used(rel->r);
    7154        3317 :                 rel_dce_sub(sql, rel->r);
    7155        3317 :                 return rel;
    7156             : 
    7157        6317 :         case op_update:
    7158             :         case op_delete:
    7159             : 
    7160        6317 :                 if (skip_proj && rel->r)
    7161        5234 :                         rel->r = rel_dce_down(sql, rel->r, 0);
    7162        6317 :                 if (!skip_proj)
    7163         980 :                         rel_dce_sub(sql, rel);
    7164             :                 return rel;
    7165             : 
    7166      331450 :         case op_topn:
    7167             :         case op_sample:
    7168             :         case op_project:
    7169             :         case op_groupby:
    7170             : 
    7171      331450 :                 if (skip_proj && rel->l)
    7172      194478 :                         rel->l = rel_dce_down(sql, rel->l, is_topn(rel->op) || is_sample(rel->op));
    7173      331450 :                 if (!skip_proj)
    7174      131407 :                         rel_dce_sub(sql, rel);
    7175             :                 return rel;
    7176             : 
    7177       41226 :         case op_union:
    7178             :         case op_inter:
    7179             :         case op_except:
    7180       41226 :                 if (skip_proj) {
    7181       21657 :                         if (rel->l)
    7182       21657 :                                 rel->l = rel_dce_down(sql, rel->l, 0);
    7183       21658 :                         if (rel->r)
    7184       21658 :                                 rel->r = rel_dce_down(sql, rel->r, 0);
    7185             :                 }
    7186       41227 :                 if (!skip_proj)
    7187       19569 :                         rel_dce_sub(sql, rel);
    7188             :                 return rel;
    7189             : 
    7190       90986 :         case op_select:
    7191       90986 :                 if (rel->l)
    7192       90986 :                         rel->l = rel_dce_down(sql, rel->l, 0);
    7193             :                 return rel;
    7194             : 
    7195      141569 :         case op_join:
    7196             :         case op_left:
    7197             :         case op_right:
    7198             :         case op_full:
    7199             :         case op_semi:
    7200             :         case op_anti:
    7201             :         case op_merge:
    7202      141569 :                 if (rel->l)
    7203      141569 :                         rel->l = rel_dce_down(sql, rel->l, 0);
    7204      141569 :                 if (rel->r)
    7205      141569 :                         rel->r = rel_dce_down(sql, rel->r, 0);
    7206             :                 return rel;
    7207             : 
    7208      310181 :         case op_ddl:
    7209      310181 :                 if (rel->flag == ddl_output || rel->flag == ddl_create_seq || rel->flag == ddl_alter_seq || rel->flag == ddl_alter_table || rel->flag == ddl_create_table || rel->flag == ddl_create_view) {
    7210       25610 :                         if (rel->l)
    7211       25274 :                                 rel->l = rel_dce_down(sql, rel->l, 0);
    7212      284571 :                 } else if (rel->flag == ddl_list || rel->flag == ddl_exception) {
    7213         489 :                         if (rel->l)
    7214         482 :                                 rel->l = rel_dce_down(sql, rel->l, 0);
    7215         489 :                         if (rel->r)
    7216         458 :                                 rel->r = rel_dce_down(sql, rel->r, 0);
    7217             :                 }
    7218             :                 return rel;
    7219             :         }
    7220             :         return rel;
    7221             : }
    7222             : 
    7223             : /* DCE
    7224             :  *
    7225             :  * Based on top relation expressions mark sub expressions as used.
    7226             :  * Then recurse down until the projections. Clean them up and repeat.
    7227             :  */
    7228             : 
    7229             : static sql_rel *
    7230      779093 : rel_dce_sub(mvc *sql, sql_rel *rel)
    7231             : {
    7232      779093 :         if (!rel)
    7233             :                 return rel;
    7234             : 
    7235             :         /*
    7236             :          * Mark used up until the next project
    7237             :          * For setops we need to first mark, then remove
    7238             :          * because of positional dependency
    7239             :          */
    7240      779091 :         rel_mark_used(sql, rel, 1);
    7241      779091 :         rel = rel_remove_unused(sql, rel);
    7242      779091 :         rel_dce_down(sql, rel, 1);
    7243      779091 :         return rel;
    7244             : }
    7245             : 
    7246             : /* add projects under set ops */
    7247             : static sql_rel *
    7248     2271569 : rel_add_projects(mvc *sql, sql_rel *rel)
    7249             : {
    7250     2271569 :         if (!rel)
    7251             :                 return rel;
    7252             : 
    7253     2271567 :         switch(rel->op) {
    7254             :         case op_basetable:
    7255             :         case op_truncate:
    7256             :                 return rel;
    7257        8654 :         case op_insert:
    7258             :         case op_update:
    7259             :         case op_delete:
    7260        8654 :                 if (rel->r)
    7261        8551 :                         rel->r = rel_add_projects(sql, rel->r);
    7262             :                 return rel;
    7263       85303 :         case op_union:
    7264             :         case op_inter:
    7265             :         case op_except:
    7266             :                 /* We can only reduce the list of expressions of an set op
    7267             :                  * if the projection under it can also be reduced.
    7268             :                  */
    7269       85303 :                 if (rel->l) {
    7270             :                         sql_rel *l = rel->l;
    7271             : 
    7272       85303 :                         if (!is_project(l->op) && !need_distinct(rel))
    7273         113 :                                 l = rel_project(sql->sa, l, rel_projections(sql, l, NULL, 1, 1));
    7274       85303 :                         rel->l = rel_add_projects(sql, l);
    7275             :                 }
    7276       85303 :                 if (rel->r) {
    7277             :                         sql_rel *r = rel->r;
    7278             : 
    7279       85303 :                         if (!is_project(r->op) && !need_distinct(rel))
    7280         152 :                                 r = rel_project(sql->sa, r, rel_projections(sql, r, NULL, 1, 1));
    7281       85303 :                         rel->r = rel_add_projects(sql, r);
    7282             :                 }
    7283             :                 return rel;
    7284      861416 :         case op_topn:
    7285             :         case op_sample:
    7286             :         case op_project:
    7287             :         case op_groupby:
    7288             :         case op_select:
    7289             :         case op_table:
    7290      861416 :                 if (rel->l && (rel->op != op_table || rel->flag != TRIGGER_WRAPPER))
    7291      835358 :                         rel->l = rel_add_projects(sql, rel->l);
    7292             :                 return rel;
    7293      422682 :         case op_join:
    7294             :         case op_left:
    7295             :         case op_right:
    7296             :         case op_full:
    7297             :         case op_semi:
    7298             :         case op_anti:
    7299             :         case op_merge:
    7300      422682 :                 if (rel->l)
    7301      422682 :                         rel->l = rel_add_projects(sql, rel->l);
    7302      422682 :                 if (rel->r)
    7303      422682 :                         rel->r = rel_add_projects(sql, rel->r);
    7304             :                 return rel;
    7305      311340 :         case op_ddl:
    7306      311340 :                 if (rel->flag == ddl_output || rel->flag == ddl_create_seq || rel->flag == ddl_alter_seq || rel->flag == ddl_alter_table || rel->flag == ddl_create_table || rel->flag == ddl_create_view) {
    7307       26769 :                         if (rel->l)
    7308       26432 :                                 rel->l = rel_add_projects(sql, rel->l);
    7309      284571 :                 } else if (rel->flag == ddl_list || rel->flag == ddl_exception) {
    7310         489 :                         if (rel->l)
    7311         482 :                                 rel->l = rel_add_projects(sql, rel->l);
    7312         489 :                         if (rel->r)
    7313         458 :                                 rel->r = rel_add_projects(sql, rel->r);
    7314             :                 }
    7315             :                 return rel;
    7316             :         }
    7317             :         return rel;
    7318             : }
    7319             : 
    7320             : sql_rel *
    7321      384318 : rel_dce(mvc *sql, sql_rel *rel)
    7322             : {
    7323      384318 :         list *refs = sa_list(sql->sa);
    7324             : 
    7325      384318 :         rel_dce_refs(sql, rel, refs);
    7326      384318 :         if (refs) {
    7327             :                 node *n;
    7328             : 
    7329      394740 :                 for(n = refs->h; n; n = n->next) {
    7330       10422 :                         sql_rel *i = n->data;
    7331             : 
    7332       10422 :                         while (!rel_is_ref(i) && i->l && !is_base(i->op))
    7333             :                                 i = i->l;
    7334             :                         if (i)
    7335       10422 :                                 rel_used(i);
    7336             :                 }
    7337             :         }
    7338      384318 :         rel = rel_add_projects(sql, rel);
    7339      384318 :         rel_used(rel);
    7340      384318 :         rel_dce_sub(sql, rel);
    7341      384318 :         return rel;
    7342             : }
    7343             : 
    7344             : static int
    7345       12130 : index_exp(sql_exp *e, sql_idx *i)
    7346             : {
    7347       12130 :         if (e->type == e_cmp && !is_complex_exp(e->flag)) {
    7348        6638 :                 switch(i->type) {
    7349        6638 :                 case hash_idx:
    7350             :                 case oph_idx:
    7351        6638 :                         if (e->flag == cmp_equal)
    7352             :                                 return 0;
    7353             :                         /* fall through */
    7354             :                 case join_idx:
    7355             :                 default:
    7356        1042 :                         return -1;
    7357             :                 }
    7358             :         }
    7359             :         return -1;
    7360             : }
    7361             : 
    7362             : static sql_idx *
    7363     2527385 : find_index(sql_allocator *sa, sql_rel *rel, sql_rel *sub, list **EXPS)
    7364             : {
    7365             :         node *n;
    7366             : 
    7367             :         /* any (partial) match of the expressions with the index columns */
    7368             :         /* Depending on the index type we may need full matches and only
    7369             :            limited number of cmp types (hash only equality etc) */
    7370             :         /* Depending on the index type we should (in the rel_bin) generate
    7371             :            more code, ie for spatial index add post filter etc, for hash
    7372             :            compute hash value and use index */
    7373             : 
    7374     2527385 :         if (sub->exps && rel->exps)
    7375     9399753 :         for(n = sub->exps->h; n; n = n->next) {
    7376             :                 prop *p;
    7377     7319101 :                 sql_exp *e = n->data;
    7378             : 
    7379     7319101 :                 if ((p = find_prop(e->p, PROP_HASHIDX)) != NULL) {
    7380             :                         list *exps, *cols;
    7381        9466 :                         sql_idx *i = p->value;
    7382             :                         fcmp cmp = (fcmp)&sql_column_kc_cmp;
    7383             : 
    7384             :                         /* join indices are only interesting for joins */
    7385        9466 :                         if (i->type == join_idx || list_length(i->columns) <= 1)
    7386           0 :                                 continue;
    7387             :                         /* based on the index type, find qualifying exps */
    7388        9466 :                         exps = list_select(rel->exps, i, (fcmp) &index_exp, (fdup)NULL);
    7389        9466 :                         if (list_empty(exps))
    7390        4857 :                                 continue;
    7391             :                         /* now we obtain the columns, move into sql_column_kc_cmp! */
    7392        4609 :                         cols = list_map(exps, sub, (fmap) &sjexp_col);
    7393             : 
    7394             :                         /* TODO check that at most 2 relations are involved */
    7395             : 
    7396             :                         /* Match the index columns with the expression columns.
    7397             :                            TODO, Allow partial matches ! */
    7398        4609 :                         if (list_match(cols, i->columns, cmp) == 0) {
    7399             :                                 /* re-order exps in index order */
    7400             :                                 node *n, *m;
    7401         256 :                                 list *es = sa_list(sa);
    7402             : 
    7403         857 :                                 for(n = i->columns->h; n; n = n->next) {
    7404             :                                         int i = 0;
    7405        2029 :                                         for(m = cols->h; m; m = m->next, i++) {
    7406        2029 :                                                 if (cmp(m->data, n->data) == 0){
    7407         601 :                                                         sql_exp *e = list_fetch(exps, i);
    7408         601 :                                                         list_append(es, e);
    7409         601 :                                                         break;
    7410             :                                                 }
    7411             :                                         }
    7412             :                                 }
    7413             :                                 /* fix the destroy function */
    7414         256 :                                 cols->destroy = NULL;
    7415         256 :                                 *EXPS = es;
    7416         256 :                                 e->used = 1;
    7417         256 :                                 return i;
    7418             :                         }
    7419        4353 :                         cols->destroy = NULL;
    7420             :                 }
    7421             :         }
    7422             :         return NULL;
    7423             : }
    7424             : 
    7425             : static inline sql_rel *
    7426     1570567 : rel_use_index(visitor *v, sql_rel *rel)
    7427             : {
    7428     1570567 :         list *exps = NULL;
    7429     1570567 :         sql_idx *i = find_index(v->sql->sa, rel, rel->l, &exps);
    7430             :         int left = 1;
    7431             : 
    7432     1570566 :         assert(is_select(rel->op) || is_join(rel->op));
    7433     1570566 :         if (!i && is_join(rel->op)) {
    7434             :                 left = 0;
    7435      956818 :                 i = find_index(v->sql->sa, rel, rel->r, &exps);
    7436             :         }
    7437             : 
    7438     1570566 :         if (i) {
    7439             :                 prop *p;
    7440             :                 node *n;
    7441             :                 int single_table = 1;
    7442             :                 sql_exp *re = NULL;
    7443             : 
    7444         852 :                 for( n = exps->h; n && single_table; n = n->next) {
    7445         596 :                         sql_exp *e = n->data;
    7446         596 :                         sql_exp *nre = e->r;
    7447             : 
    7448         596 :                         if (is_join(rel->op) && ((left && !rel_find_exp(rel->l, e->l)) || (!left && !rel_find_exp(rel->r, e->l))))
    7449          45 :                                 nre = e->l;
    7450         596 :                         single_table = (!re || (exp_relname(nre) && exp_relname(re) && strcmp(exp_relname(nre), exp_relname(re)) == 0));
    7451             :                         re = nre;
    7452             :                 }
    7453         256 :                 if (single_table) { /* add PROP_HASHCOL to all column exps */
    7454         672 :                         for( n = exps->h; n; n = n->next) {
    7455         476 :                                 sql_exp *e = n->data;
    7456         476 :                                 int anti = is_anti(e), semantics = is_semantics(e);
    7457             : 
    7458             :                                 /* swapped ? */
    7459         476 :                                 if (is_join(rel->op) && ((left && !rel_find_exp(rel->l, e->l)) || (!left && !rel_find_exp(rel->r, e->l))))
    7460          41 :                                         n->data = e = exp_compare(v->sql->sa, e->r, e->l, cmp_equal);
    7461         476 :                                 if (anti) set_anti(e);
    7462         476 :                                 if (semantics) set_semantics(e);
    7463         476 :                                 p = find_prop(e->p, PROP_HASHCOL);
    7464         476 :                                 if (!p)
    7465         215 :                                         e->p = p = prop_create(v->sql->sa, PROP_HASHCOL, e->p);
    7466         476 :                                 p->value = i;
    7467             :                         }
    7468             :                 }
    7469             :                 /* add the remaining exps to the new exp list */
    7470         256 :                 if (list_length(rel->exps) > list_length(exps)) {
    7471          40 :                         for( n = rel->exps->h; n; n = n->next) {
    7472          30 :                                 sql_exp *e = n->data;
    7473          30 :                                 if (!list_find(exps, e, (fcmp)&exp_cmp))
    7474          26 :                                         list_append(exps, e);
    7475             :                         }
    7476             :                 }
    7477         256 :                 rel->exps = exps;
    7478             :         }
    7479     1570566 :         return rel;
    7480             : }
    7481             : 
    7482             : static int
    7483      153044 : score_se_base(visitor *v, sql_rel *rel, sql_exp *e)
    7484             : {
    7485             :         int res = 0;
    7486      153044 :         sql_subtype *t = exp_subtype(e);
    7487             :         sql_column *c = NULL;
    7488             : 
    7489             :         /* can we find out if the underlying table is sorted */
    7490      153044 :         if ((c = exp_find_column(rel, e, -2)) && v->storage_based_opt && mvc_is_sorted(v->sql, c))
    7491             :                 res += 600;
    7492             : 
    7493             :         /* prefer the shorter var types over the longer ones */
    7494      306088 :         res += sql_class_base_score(v, c, t, is_equality_or_inequality_exp(e->flag)); /* smaller the type, better */
    7495      153044 :         return res;
    7496             : }
    7497             : 
    7498             : static int
    7499      254908 : score_se(visitor *v, sql_rel *rel, sql_exp *e)
    7500             : {
    7501             :         int score = 0;
    7502      254908 :         if (e->type == e_cmp && !is_complex_exp(e->flag)) {
    7503      153044 :                 sql_exp *l = e->l;
    7504             : 
    7505      153044 :                 while (l->type == e_cmp) { /* go through nested comparisons */
    7506             :                         sql_exp *ll;
    7507             : 
    7508           4 :                         if (l->flag == cmp_filter || l->flag == cmp_or)
    7509           0 :                                 ll = ((list*)l->l)->h->data;
    7510             :                         else
    7511           4 :                                 ll = l->l;
    7512           4 :                         if (ll->type != e_cmp)
    7513             :                                 break;
    7514             :                         l = ll;
    7515             :                 }
    7516      153044 :                 score += score_se_base(v, rel, l);
    7517             :         }
    7518      254908 :         score += exp_keyvalue(e);
    7519      254908 :         return score;
    7520             : }
    7521             : 
    7522             : static inline sql_rel *
    7523     4828567 : rel_select_order(visitor *v, sql_rel *rel)
    7524             : {
    7525             :         int *scores = NULL;
    7526             :         sql_exp **exps = NULL;
    7527             : 
    7528     4828567 :         if (is_select(rel->op) && list_length(rel->exps) > 1) {
    7529             :                 node *n;
    7530      123181 :                 int i, nexps = list_length(rel->exps);
    7531      123181 :                 scores = SA_NEW_ARRAY(v->sql->ta, int, nexps);
    7532      123181 :                 exps = SA_NEW_ARRAY(v->sql->ta, sql_exp*, nexps);
    7533             : 
    7534      378089 :                 for (i = 0, n = rel->exps->h; n; i++, n = n->next) {
    7535      254908 :                         exps[i] = n->data;
    7536      254908 :                         scores[i] = score_se(v, rel, n->data);
    7537             :                 }
    7538      123181 :                 GDKqsort(scores, exps, NULL, nexps, sizeof(int), sizeof(void *), TYPE_int, true, true);
    7539             : 
    7540      378089 :                 for (i = 0, n = rel->exps->h; n; i++, n = n->next)
    7541      254908 :                         n->data = exps[i];
    7542             :         }
    7543             : 
    7544     4828567 :         return rel;
    7545             : }
    7546             : 
    7547             : static inline sql_rel *
    7548      523567 : rel_simplify_like_select(visitor *v, sql_rel *rel)
    7549             : {
    7550      523567 :         if (is_select(rel->op) && !list_empty(rel->exps)) {
    7551      617237 :                 for (node *n = rel->exps->h; n; n = n->next) {
    7552      328862 :                         sql_exp *e = n->data;
    7553      328862 :                         list *l = e->l;
    7554      328862 :                         list *r = e->r;
    7555             : 
    7556      328862 :                         if (e->type == e_cmp && e->flag == cmp_filter && strcmp(((sql_subfunc*)e->f)->func->base.name, "like") == 0 && list_length(l) == 1 && list_length(r) == 3) {
    7557         460 :                                 list *r = e->r;
    7558         460 :                                 sql_exp *fmt = r->h->data;
    7559         460 :                                 sql_exp *esc = r->h->next->data;
    7560         460 :                                 sql_exp *isen = r->h->next->next->data;
    7561             :                                 int rewrite = 0, isnull = 0;
    7562             : 
    7563         460 :                                 if (fmt->type == e_convert)
    7564          41 :                                         fmt = fmt->l;
    7565             :                                 /* check for simple like expression */
    7566         460 :                                 if (exp_is_null(fmt)) {
    7567             :                                         isnull = 1;
    7568         458 :                                 } else if (is_atom(fmt->type)) {
    7569             :                                         atom *fa = NULL;
    7570             : 
    7571         417 :                                         if (fmt->l)
    7572             :                                                 fa = fmt->l;
    7573         417 :                                         if (fa && fa->data.vtype == TYPE_str && !strchr(fa->data.val.sval, '%') && !strchr(fa->data.val.sval, '_'))
    7574             :                                                 rewrite = 1;
    7575             :                                 }
    7576         460 :                                 if (rewrite && !isnull) { /* check escape flag */
    7577          41 :                                         if (exp_is_null(esc)) {
    7578             :                                                 isnull = 1;
    7579             :                                         } else {
    7580          41 :                                                 atom *ea = esc->l;
    7581             : 
    7582          41 :                                                 if (!is_atom(esc->type) || !ea)
    7583             :                                                         rewrite = 0;
    7584          41 :                                                 else if (ea->data.vtype != TYPE_str || strlen(ea->data.val.sval) != 0)
    7585             :                                                         rewrite = 0;
    7586             :                                         }
    7587             :                                 }
    7588         460 :                                 if (rewrite && !isnull) { /* check insensitive flag */
    7589          36 :                                         if (exp_is_null(isen)) {
    7590             :                                                 isnull = 1;
    7591             :                                         } else {
    7592          36 :                                                 atom *ia = isen->l;
    7593             : 
    7594          36 :                                                 if (!is_atom(isen->type) || !ia)
    7595             :                                                         rewrite = 0;
    7596          36 :                                                 else if (ia->data.vtype != TYPE_bit || ia->data.val.btval == 1)
    7597             :                                                         rewrite = 0;
    7598             :                                         }
    7599             :                                 }
    7600         460 :                                 if (isnull) {
    7601           2 :                                         rel->exps = list_append(sa_list(v->sql->sa), exp_null(v->sql->sa, sql_bind_localtype("bit")));
    7602           2 :                                         v->changes++;
    7603           2 :                                         return rel;
    7604         458 :                                 } else if (rewrite) {   /* rewrite to cmp_equal ! */
    7605          29 :                                         list *l = e->l;
    7606          29 :                                         list *r = e->r;
    7607          54 :                                         n->data = exp_compare(v->sql->sa, l->h->data, r->h->data, is_anti(e) ? cmp_notequal : cmp_equal);
    7608          29 :                                         v->changes++;
    7609             :                                 }
    7610             :                         }
    7611             :                 }
    7612             :         }
    7613             :         return rel;
    7614             : }
    7615             : 
    7616             : static sql_exp *
    7617     9490130 : rel_simplify_predicates(visitor *v, sql_rel *rel, sql_exp *e, int depth)
    7618             : {
    7619             :         (void)depth;
    7620     9490130 :         if (is_func(e->type) && list_length(e->l) == 3 && is_case_func((sql_subfunc*)e->f) /*is_ifthenelse_func((sql_subfunc*)e->f)*/) {
    7621       27360 :                 list *args = e->l;
    7622       27360 :                 sql_exp *ie = args->h->data;
    7623             : 
    7624       27360 :                 if (exp_is_true(ie)) { /* ifthenelse(true, x, y) -> x */
    7625          13 :                         sql_exp *res = args->h->next->data;
    7626          13 :                         if (exp_name(e))
    7627           5 :                                 exp_prop_alias(v->sql->sa, res, e);
    7628          13 :                         v->changes++;
    7629          13 :                         return res;
    7630       27347 :                 } else if (exp_is_false(ie) || exp_is_null(ie)) { /* ifthenelse(false or null, x, y) -> y */
    7631           7 :                         sql_exp *res = args->h->next->next->data;
    7632           7 :                         if (exp_name(e))
    7633           2 :                                 exp_prop_alias(v->sql->sa, res, e);
    7634           7 :                         v->changes++;
    7635           7 :                         return res;
    7636             :                 }
    7637             :         }
    7638     9490110 :         if (is_select(rel->op) || is_join(rel->op) || is_semi(rel->op)) {
    7639     2292743 :                 if (is_compare(e->type) && is_semantics(e) && (e->flag == cmp_equal || e->flag == cmp_notequal) && exp_is_null(e->r)) {
    7640             :                         /* simplify 'is null' predicates on constants */
    7641       20449 :                         if (exp_is_null(e->l)) {
    7642          32 :                                 int nval = e->flag == cmp_equal;
    7643          32 :                                 if (is_anti(e)) nval = !nval;
    7644          32 :                                 e = exp_atom_bool(v->sql->sa, nval);
    7645          32 :                                 v->changes++;
    7646          32 :                                 return e;
    7647       20417 :                         } else if (exp_is_not_null(e->l)) {
    7648         635 :                                 int nval = e->flag == cmp_notequal;
    7649         635 :                                 if (is_anti(e)) nval = !nval;
    7650         635 :                                 e = exp_atom_bool(v->sql->sa, nval);
    7651         635 :                                 v->changes++;
    7652         635 :                                 return e;
    7653             :                         }
    7654             :                 }
    7655     2292076 :                 if (is_atom(e->type) && ((!e->l && !e->r && !e->f) || e->r)) /* prepared statement parameter or argument */
    7656             :                         return e;
    7657     2291195 :                 if (is_atom(e->type) && e->l) { /* direct literal */
    7658             :                         atom *a = e->l;
    7659      481081 :                         int flag = a->data.val.bval;
    7660             : 
    7661             :                         /* remove simple select true expressions */
    7662      481081 :                         if (flag)
    7663             :                                 return e;
    7664             :                 }
    7665     1852510 :                 if (is_compare(e->type) && is_theta_exp(e->flag)) {
    7666      598700 :                         sql_exp *l = e->l;
    7667      598700 :                         sql_exp *r = e->r;
    7668             : 
    7669      598700 :                         if (is_func(l->type) && (e->flag == cmp_equal || e->flag == cmp_notequal)) {
    7670       11705 :                                 sql_subfunc *f = l->f;
    7671             : 
    7672             :                                 /* rewrite isnull(x) = TRUE/FALSE => x =/<> NULL */
    7673       11705 :                                 if (!f->func->s && is_isnull_func(f)) {
    7674        1671 :                                         list *args = l->l;
    7675        1671 :                                         sql_exp *ie = args->h->data;
    7676             : 
    7677        1671 :                                         if (!has_nil(ie) || exp_is_not_null(ie)) { /* is null on something that is never null, is always false */
    7678         722 :                                                 ie = exp_atom_bool(v->sql->sa, 0);
    7679         722 :                                                 v->changes++;
    7680         722 :                                                 e->l = ie;
    7681         949 :                                         } else if (exp_is_null(ie)) { /* is null on something that is always null, is always true */
    7682           0 :                                                 ie = exp_atom_bool(v->sql->sa, 1);
    7683           0 :                                                 v->changes++;
    7684           0 :                                                 e->l = ie;
    7685         949 :                                         } else if (is_atom(r->type) && r->l) { /* direct literal */
    7686             :                                                 atom *a = r->l;
    7687             : 
    7688         949 :                                                 if (a->isnull) {
    7689           0 :                                                         if (is_semantics(e)) { /* isnull(x) = NULL -> false, isnull(x) <> NULL -> true */
    7690           0 :                                                                 int flag = e->flag == cmp_notequal;
    7691           0 :                                                                 if (is_anti(e))
    7692           0 :                                                                         flag = !flag;
    7693           0 :                                                                 e = exp_atom_bool(v->sql->sa, flag);
    7694             :                                                         } else /* always NULL */
    7695           0 :                                                                 e = exp_null(v->sql->sa, sql_bind_localtype("bit"));
    7696           0 :                                                         v->changes++;
    7697             :                                                 } else {
    7698         949 :                                                         int flag = a->data.val.bval;
    7699             : 
    7700         949 :                                                         assert(list_length(args) == 1);
    7701         949 :                                                         l = args->h->data;
    7702         949 :                                                         if (exp_subtype(l)) {
    7703         949 :                                                                 r = exp_atom(v->sql->sa, atom_general(v->sql->sa, exp_subtype(l), NULL));
    7704         949 :                                                                 e = exp_compare(v->sql->sa, l, r, e->flag);
    7705         949 :                                                                 if (e && !flag)
    7706         350 :                                                                         set_anti(e);
    7707         949 :                                                                 if (e)
    7708         949 :                                                                         set_semantics(e);
    7709         949 :                                                                 v->changes++;
    7710             :                                                         }
    7711             :                                                 }
    7712             :                                         }
    7713       10034 :                                 } else if (!f->func->s && is_not_func(f)) {
    7714           2 :                                         if (is_atom(r->type) && r->l) { /* direct literal */
    7715             :                                                 atom *a = r->l;
    7716           2 :                                                 list *args = l->l;
    7717           2 :                                                 sql_exp *inner = args->h->data;
    7718           2 :                                                 sql_subfunc *inf = inner->f;
    7719             : 
    7720           2 :                                                 assert(list_length(args) == 1);
    7721             : 
    7722             :                                                 /* not(not(x)) = TRUE/FALSE => x = TRUE/FALSE */
    7723           2 :                                                 if (is_func(inner->type) &&
    7724           0 :                                                         !inf->func->s &&
    7725           0 :                                                         is_not_func(inf)) {
    7726           0 :                                                         int anti = is_anti(e), is_semantics = is_semantics(e);
    7727             : 
    7728           0 :                                                         args = inner->l;
    7729           0 :                                                         assert(list_length(args) == 1);
    7730           0 :                                                         l = args->h->data;
    7731           0 :                                                         e = exp_compare(v->sql->sa, l, r, e->flag);
    7732           0 :                                                         if (anti) set_anti(e);
    7733           0 :                                                         if (is_semantics) set_semantics(e);
    7734           0 :                                                         v->changes++;
    7735             :                                                 /* rewrite not(=/<>(a,b)) = TRUE/FALSE => a=b / a<>b */
    7736           2 :                                                 } else if (is_func(inner->type) &&
    7737           0 :                                                         !inf->func->s &&
    7738           0 :                                                         (!strcmp(inf->func->base.name, "=") ||
    7739           0 :                                                          !strcmp(inf->func->base.name, "<>"))) {
    7740           0 :                                                         int flag = a->data.val.bval;
    7741             :                                                         sql_exp *ne;
    7742           0 :                                                         args = inner->l;
    7743             : 
    7744           0 :                                                         if (!strcmp(inf->func->base.name, "<>"))
    7745           0 :                                                                 flag = !flag;
    7746           0 :                                                         if (e->flag == cmp_notequal)
    7747           0 :                                                                 flag = !flag;
    7748           0 :                                                         assert(list_length(args) == 2);
    7749           0 :                                                         l = args->h->data;
    7750           0 :                                                         r = args->h->next->data;
    7751           0 :                                                         ne = exp_compare(v->sql->sa, l, r, (!flag)?cmp_equal:cmp_notequal);
    7752           0 :                                                         if (a->isnull)
    7753           0 :                                                                 e->l = ne;
    7754             :                                                         else
    7755             :                                                                 e = ne;
    7756           0 :                                                         v->changes++;
    7757           2 :                                                 } else if (a && a->data.vtype == TYPE_bit) {
    7758           2 :                                                         int anti = is_anti(e), is_semantics = is_semantics(e);
    7759             : 
    7760             :                                                         /* change atom's value on right */
    7761           2 :                                                         l = args->h->data;
    7762           2 :                                                         if (!a->isnull)
    7763           2 :                                                                 a->data.val.bval = !a->data.val.bval;
    7764           2 :                                                         e = exp_compare(v->sql->sa, l, r, e->flag);
    7765           2 :                                                         if (anti) set_anti(e);
    7766           2 :                                                         if (is_semantics) set_semantics(e);
    7767           2 :                                                         v->changes++;
    7768             :                                                 }
    7769             :                                         }
    7770             :                                 }
    7771      586995 :                         } else if (is_atom(l->type) && is_atom(r->type) && !is_semantics(e) && !e->f) {
    7772             :                                 /* compute comparisons on atoms */
    7773        1145 :                                 if (exp_is_null(l) || exp_is_null(r)) {
    7774          32 :                                         e = exp_null(v->sql->sa, sql_bind_localtype("bit"));
    7775          32 :                                         v->changes++;
    7776        1113 :                                 } else if (l->l && r->l) {
    7777        1113 :                                         int res = atom_cmp(l->l, r->l);
    7778        1113 :                                         bool flag = !is_anti(e);
    7779             : 
    7780        1113 :                                         if (res == 0)
    7781         416 :                                                 e = exp_atom_bool(v->sql->sa, (e->flag == cmp_equal || e->flag == cmp_gte || e->flag == cmp_lte) ? flag : !flag);
    7782         697 :                                         else if (res > 0)
    7783          37 :                                                 e = exp_atom_bool(v->sql->sa, (e->flag == cmp_gt || e->flag == cmp_gte || e->flag == cmp_notequal) ? flag : !flag);
    7784             :                                         else
    7785         660 :                                                 e = exp_atom_bool(v->sql->sa, (e->flag == cmp_lt || e->flag == cmp_lte || e->flag == cmp_notequal) ? flag : !flag);
    7786        1113 :                                         v->changes++;
    7787             :                                 }
    7788             :                         }
    7789             :                 }
    7790             :         }
    7791             :         return e;
    7792             : }
    7793             : 
    7794             : static void split_exps(mvc *sql, list *exps, sql_rel *rel);
    7795             : 
    7796             : static int
    7797      282052 : exp_match_exp_cmp( sql_exp *e1, sql_exp *e2)
    7798             : {
    7799      282052 :         if (exp_match_exp(e1,e2))
    7800        3692 :                 return 0;
    7801             :         return -1;
    7802             : }
    7803             : 
    7804             : static int
    7805      266403 : exp_refers_cmp( sql_exp *e1, sql_exp *e2)
    7806             : {
    7807      266403 :         if (exp_refers(e1,e2))
    7808           0 :                 return 0;
    7809             :         return -1;
    7810             : }
    7811             : 
    7812             : static sql_exp *
    7813       39557 : add_exp_too_project(mvc *sql, sql_exp *e, sql_rel *rel)
    7814             : {
    7815       39557 :         node *n = list_find(rel->exps, e, (fcmp)&exp_match_exp_cmp);
    7816             : 
    7817             :         /* if not matching we may refer to an older expression */
    7818       39557 :         if (!n)
    7819       35865 :                 n = list_find(rel->exps, e, (fcmp)&exp_refers_cmp);
    7820       39557 :         if (!n) {
    7821       35865 :                 exp_label(sql->sa, e, ++sql->label);
    7822       35865 :                 append(rel->exps, e);
    7823             :         } else {
    7824        3692 :                 sql_exp *ne = n->data;
    7825             : 
    7826        3692 :                 if (rel && rel->l) {
    7827        7384 :                         if ((exp_relname(ne) && exp_name(ne) && rel_bind_column2(sql, rel->l, exp_relname(ne), exp_name(ne), 0)) ||
    7828        3692 :                            (!exp_relname(ne) && exp_name(ne) && rel_bind_column(sql, rel->l, exp_name(ne), 0, 1))) {
    7829           0 :                                 exp_label(sql->sa, e, ++sql->label);
    7830           0 :                                 append(rel->exps, e);
    7831             :                                 ne = e;
    7832             :                         }
    7833             :                 }
    7834             :                 e = ne;
    7835             :         }
    7836       39557 :         e = exp_ref(sql, e);
    7837       39557 :         return e;
    7838             : }
    7839             : 
    7840             : static void
    7841       45438 : add_exps_too_project(mvc *sql, list *exps, sql_rel *rel)
    7842             : {
    7843             :         node *n;
    7844             : 
    7845       45438 :         if (!exps)
    7846             :                 return;
    7847      134776 :         for(n=exps->h; n; n = n->next) {
    7848       89338 :                 sql_exp *e = n->data;
    7849             : 
    7850       89338 :                 if (e->type != e_column && !exp_is_atom(e))
    7851       39371 :                         n->data = add_exp_too_project(sql, e, rel);
    7852             :         }
    7853             : }
    7854             : 
    7855             : static sql_exp *
    7856      256405 : split_exp(mvc *sql, sql_exp *e, sql_rel *rel)
    7857             : {
    7858      256405 :         if (exp_is_atom(e))
    7859             :                 return e;
    7860      234593 :         switch(e->type) {
    7861             :         case e_column:
    7862             :                 return e;
    7863       21351 :         case e_convert:
    7864       21351 :                 e->l = split_exp(sql, e->l, rel);
    7865       21351 :                 return e;
    7866       54016 :         case e_aggr:
    7867             :         case e_func:
    7868       54016 :                 if (!is_analytic(e) && !exp_has_sideeffect(e)) {
    7869       50519 :                         sql_subfunc *f = e->f;
    7870       50519 :                         if (e->type == e_func && !f->func->s && is_caselike_func(f) /*is_ifthenelse_func(f)*/) {
    7871             :                                 return e;
    7872             :                         } else {
    7873       45438 :                                 split_exps(sql, e->l, rel);
    7874       45438 :                                 add_exps_too_project(sql, e->l, rel);
    7875             :                         }
    7876             :                 }
    7877             :                 return e;
    7878          12 :         case e_cmp:
    7879          12 :                 if (e->flag == cmp_or || e->flag == cmp_filter) {
    7880           0 :                         split_exps(sql, e->l, rel);
    7881           0 :                         split_exps(sql, e->r, rel);
    7882          12 :                 } else if (e->flag == cmp_in || e->flag == cmp_notin) {
    7883           0 :                         e->l = split_exp(sql, e->l, rel);
    7884           0 :                         split_exps(sql, e->r, rel);
    7885             :                 } else {
    7886          12 :                         e->l = split_exp(sql, e->l, rel);
    7887          12 :                         e->r = split_exp(sql, e->r, rel);
    7888          12 :                         if (e->f)
    7889          12 :                                 e->f = split_exp(sql, e->f, rel);
    7890             :                 }
    7891             :                 return e;
    7892             :         case e_atom:
    7893             :         case e_psm:
    7894             :                 return e;
    7895             :         }
    7896             :         return e;
    7897             : }
    7898             : 
    7899             : static void
    7900       45447 : split_exps(mvc *sql, list *exps, sql_rel *rel)
    7901             : {
    7902             :         node *n;
    7903             : 
    7904       45447 :         if (!exps)
    7905             :                 return;
    7906      134842 :         for(n=exps->h; n; n = n->next){
    7907       89395 :                 sql_exp *e = n->data;
    7908             : 
    7909       89395 :                 e = split_exp(sql, e, rel);
    7910       89395 :                 n->data = e;
    7911             :         }
    7912             : }
    7913             : 
    7914             : static sql_rel *
    7915      595625 : rel_split_project(visitor *v, sql_rel *rel, int top)
    7916             : {
    7917      595625 :         if (mvc_highwater(v->sql))
    7918           0 :                 return sql_error(v->sql, 10, SQLSTATE(42000) "Query too complex: running out of stack space");
    7919             : 
    7920      595625 :         if (!rel)
    7921             :                 return NULL;
    7922      595623 :         if (is_project(rel->op) && list_length(rel->exps) && (is_groupby(rel->op) || rel->l) && !need_distinct(rel) && !is_single(rel)) {
    7923      147816 :                 list *exps = rel->exps;
    7924             :                 node *n;
    7925             :                 int funcs = 0;
    7926             :                 sql_rel *nrel;
    7927             : 
    7928             :                 /* are there functions */
    7929     1017669 :                 for (n=exps->h; n && !funcs; n = n->next) {
    7930      869853 :                         sql_exp *e = n->data;
    7931             : 
    7932      869853 :                         funcs = exp_has_func(e);
    7933             :                 }
    7934             :                 /* introduce extra project */
    7935      147816 :                 if (funcs && rel->op != op_project) {
    7936           9 :                         nrel = rel_project(v->sql->sa, rel->l,
    7937           9 :                                 rel_projections(v->sql, rel->l, NULL, 1, 1));
    7938           9 :                         rel->l = nrel;
    7939             :                         /* recursively split all functions and add those to the projection list */
    7940           9 :                         split_exps(v->sql, rel->exps, nrel);
    7941          18 :                         if (nrel->l && !(nrel->l = rel_split_project(v, nrel->l, (is_topn(rel->op)||is_sample(rel->op))?top:0)))
    7942             :                                 return NULL;
    7943           9 :                         return rel;
    7944      147807 :                 } else if (funcs && !top && !rel->r) {
    7945             :                         /* projects can have columns point back into the expression list, ie
    7946             :                          * create a new list including the split expressions */
    7947             :                         node *n;
    7948       16139 :                         list *exps = rel->exps;
    7949             : 
    7950       16139 :                         rel->exps = sa_list(v->sql->sa);
    7951      161762 :                         for (n=exps->h; n; n = n->next)
    7952      145623 :                                 append(rel->exps, split_exp(v->sql, n->data, rel));
    7953      131668 :                 } else if (funcs && top && rel_is_ref(rel) && !rel->r) {
    7954             :                         /* inplace */
    7955           0 :                         list *exps = rel_projections(v->sql, rel, NULL, 1, 1);
    7956           0 :                         sql_rel *l = rel_project(v->sql->sa, rel->l, NULL);
    7957           0 :                         rel->l = l;
    7958           0 :                         l->exps = rel->exps;
    7959           0 :                         rel->exps = exps;
    7960             :                 }
    7961             :         }
    7962      595614 :         if (is_set(rel->op) || is_basetable(rel->op))
    7963             :                 return rel;
    7964      389313 :         if (rel->l) {
    7965      732524 :                 rel->l = rel_split_project(v, rel->l, (is_topn(rel->op)||is_sample(rel->op)||is_ddl(rel->op)||is_modify(rel->op))?top:0);
    7966      370261 :                 if (!rel->l)
    7967             :                         return NULL;
    7968             :         }
    7969      389313 :         if ((is_join(rel->op) || is_semi(rel->op)) && rel->r) {
    7970      138118 :                 rel->r = rel_split_project(v, rel->r, (is_topn(rel->op)||is_sample(rel->op)||is_ddl(rel->op)||is_modify(rel->op))?top:0);
    7971      138118 :                 if (!rel->r)
    7972           0 :                         return NULL;
    7973             :         }
    7974             :         return rel;
    7975             : }
    7976             : 
    7977             : static void select_split_exps(mvc *sql, list *exps, sql_rel *rel);
    7978             : 
    7979             : static sql_exp *
    7980       60449 : select_split_exp(mvc *sql, sql_exp *e, sql_rel *rel)
    7981             : {
    7982       60449 :         switch(e->type) {
    7983             :         case e_column:
    7984             :                 return e;
    7985        8016 :         case e_convert