LCOV - code coverage report
Current view: top level - sql/storage/bat - bat_logger.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 591 1128 52.4 %
Date: 2021-10-13 02:24:04 Functions: 23 23 100.0 %

          Line data    Source code
       1             : /*
       2             :  * This Source Code Form is subject to the terms of the Mozilla Public
       3             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       4             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       5             :  *
       6             :  * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V.
       7             :  */
       8             : 
       9             : #include "monetdb_config.h"
      10             : #include "bat_logger.h"
      11             : #include "bat_utils.h"
      12             : #include "sql_types.h" /* EC_POS */
      13             : #include "wlc.h"
      14             : #include "gdk_logger_internals.h"
      15             : #include "mutils.h"
      16             : 
      17             : #define CATALOG_NOV2019 52203   /* first in Apr2019 */
      18             : #define CATALOG_JUN2020 52204   /* first in Jun2020 */
      19             : #define CATALOG_JUN2020_MMT 52206       /* only in Jun2020-mmt */
      20             : #define CATALOG_OCT2020 52205   /* first in Oct2020 */
      21             : 
      22             : /* Note, CATALOG version 52300 is the first one where the basic system
      23             :  * tables (the ones created in store.c) have fixed and unchangeable
      24             :  * ids. */
      25             : 
      26             : /* return GDK_SUCCEED if we can handle the upgrade from oldversion to
      27             :  * newversion */
      28             : static gdk_return
      29           8 : bl_preversion(sqlstore *store, int oldversion, int newversion)
      30             : {
      31             :         (void)newversion;
      32             : 
      33             : #ifdef CATALOG_NOV2019
      34           8 :         if (oldversion == CATALOG_NOV2019) {
      35             :                 /* upgrade to default releases */
      36           0 :                 store->catalog_version = oldversion;
      37           0 :                 return GDK_SUCCEED;
      38             :         }
      39             : #endif
      40             : 
      41             : #ifdef CATALOG_JUN2020
      42           8 :         if (oldversion == CATALOG_JUN2020) {
      43             :                 /* upgrade to default releases */
      44           0 :                 store->catalog_version = oldversion;
      45           0 :                 return GDK_SUCCEED;
      46             :         }
      47             : #endif
      48             : 
      49             : #ifdef CATALOG_JUN2020_MMT
      50           8 :         if (oldversion == CATALOG_JUN2020_MMT) {
      51             :                 /* upgrade to default releases */
      52           0 :                 store->catalog_version = oldversion;
      53           0 :                 return GDK_SUCCEED;
      54             :         }
      55             : #endif
      56             : 
      57             : #ifdef CATALOG_OCT2020
      58           8 :         if (oldversion == CATALOG_OCT2020) {
      59             :                 /* upgrade to default releases */
      60           8 :                 store->catalog_version = oldversion;
      61           8 :                 return GDK_SUCCEED;
      62             :         }
      63             : #endif
      64             : 
      65             :         return GDK_FAIL;
      66             : }
      67             : 
      68             : #define N(schema, table, column)        schema "_" table "_" column
      69             : 
      70             : #define D(schema, table)        "D_" schema "_" table
      71             : 
      72             : #if defined CATALOG_JUN2020 || defined CATALOG_OCT2020
      73             : static gdk_return
      74          16 : tabins(logger *lg, old_logger *old_lg, bool first, int tt, int nid, ...)
      75             : {
      76             :         va_list va;
      77             :         int cid;
      78             :         const void *cval;
      79             :         gdk_return rc;
      80             :         BAT *b;
      81             : 
      82          16 :         va_start(va, nid);
      83         192 :         while ((cid = va_arg(va, int)) != 0) {
      84         176 :                 cval = va_arg(va, void *);
      85         176 :                 if ((b = temp_descriptor(logger_find_bat(lg, cid))) == NULL) {
      86           0 :                         va_end(va);
      87           0 :                         return GDK_FAIL;
      88             :                 }
      89         176 :                 if (first && BUNfnd(old_lg->add, &b->batCacheid) == BUN_NONE) {
      90          64 :                         BAT *bn = COLcopy(b, b->ttype, true, PERSISTENT);
      91         128 :                         if (bn == NULL ||
      92         128 :                                 BUNappend(old_lg->add, &bn->batCacheid, false) != GDK_SUCCEED ||
      93          64 :                                 BUNappend(old_lg->del, &b->batCacheid, false) != GDK_SUCCEED) {
      94           0 :                                 va_end(va);
      95           0 :                                 bat_destroy(b);
      96           0 :                                 bat_destroy(bn);
      97           0 :                                 return GDK_FAIL;
      98             :                         }
      99          64 :                         BBPretain(bn->batCacheid);
     100             :                         /* logical refs of b stay the same: it is moved from catalog_bid to del */
     101          64 :                         bat_destroy(b);
     102          64 :                         BUN p = BUNfnd(lg->catalog_id, &cid);
     103          64 :                         assert(p != BUN_NONE);
     104          64 :                         if (BUNreplace(lg->catalog_bid, p, &bn->batCacheid, false) != GDK_SUCCEED) {
     105           0 :                                 va_end(va);
     106           0 :                                 bat_destroy(bn);
     107           0 :                                 return GDK_FAIL;
     108             :                         }
     109          64 :                         BBPretain(bn->batCacheid);
     110             :                         b = bn;
     111             :                 }
     112         176 :                 rc = BUNappend(b, cval, true);
     113         176 :                 bat_destroy(b);
     114         176 :                 if (rc != GDK_SUCCEED) {
     115           0 :                         va_end(va);
     116           0 :                         return rc;
     117             :                 }
     118             :         }
     119          16 :         va_end(va);
     120             : 
     121          16 :         if (tt >= 0) {
     122           0 :                 if ((b = COLnew(0, tt, 0, PERSISTENT)) == NULL)
     123             :                         return GDK_FAIL;
     124           0 :                 rc = log_bat_persists(lg, b, nid);
     125           0 :                 bat_destroy(b);
     126           0 :                 if (rc != GDK_SUCCEED)
     127           0 :                         return rc;
     128             :         }
     129             :         return GDK_SUCCEED;
     130             : }
     131             : #endif
     132             : 
     133             : struct table {
     134             :         const char *schema;
     135             :         const char *table;
     136             :         const char *column;
     137             :         const char *fullname;
     138             :         int newid;
     139             :         bool hasids;
     140             : } tables[] = {
     141             :         {
     142             :                 .schema = "sys",
     143             :                 .newid = 2000,
     144             :         },
     145             :         {
     146             :                 .schema = "sys",
     147             :                 .table = "schemas",
     148             :                 .fullname = "D_sys_schemas",
     149             :                 .newid = 2001,
     150             :         },
     151             :         {
     152             :                 .schema = "sys",
     153             :                 .table = "schemas",
     154             :                 .column = "id",
     155             :                 .fullname = "sys_schemas_id",
     156             :                 .newid = 2002,
     157             :                 .hasids = true,
     158             :         },
     159             :         {
     160             :                 .schema = "sys",
     161             :                 .table = "schemas",
     162             :                 .column = "name",
     163             :                 .fullname = "sys_schemas_name",
     164             :                 .newid = 2003,
     165             :         },
     166             :         {
     167             :                 .schema = "sys",
     168             :                 .table = "schemas",
     169             :                 .column = "authorization",
     170             :                 .fullname = "sys_schemas_authorization",
     171             :                 .newid = 2004,
     172             :         },
     173             :         {
     174             :                 .schema = "sys",
     175             :                 .table = "schemas",
     176             :                 .column = "owner",
     177             :                 .fullname = "sys_schemas_owner",
     178             :                 .newid = 2005,
     179             :         },
     180             :         {
     181             :                 .schema = "sys",
     182             :                 .table = "schemas",
     183             :                 .column = "system",
     184             :                 .fullname = "sys_schemas_system",
     185             :                 .newid = 2006,
     186             :         },
     187             :         {
     188             :                 .schema = "sys",
     189             :                 .table = "types",
     190             :                 .fullname = "D_sys_types",
     191             :                 .newid = 2007,
     192             :         },
     193             :         {
     194             :                 .schema = "sys",
     195             :                 .table = "types",
     196             :                 .column = "id",
     197             :                 .fullname = "sys_types_id",
     198             :                 .newid = 2008,
     199             :         },
     200             :         {
     201             :                 .schema = "sys",
     202             :                 .table = "types",
     203             :                 .column = "systemname",
     204             :                 .fullname = "sys_types_systemname",
     205             :                 .newid = 2009,
     206             :         },
     207             :         {
     208             :                 .schema = "sys",
     209             :                 .table = "types",
     210             :                 .column = "sqlname",
     211             :                 .fullname = "sys_types_sqlname",
     212             :                 .newid = 2010,
     213             :         },
     214             :         {
     215             :                 .schema = "sys",
     216             :                 .table = "types",
     217             :                 .column = "digits",
     218             :                 .fullname = "sys_types_digits",
     219             :                 .newid = 2011,
     220             :         },
     221             :         {
     222             :                 .schema = "sys",
     223             :                 .table = "types",
     224             :                 .column = "scale",
     225             :                 .fullname = "sys_types_scale",
     226             :                 .newid = 2012,
     227             :         },
     228             :         {
     229             :                 .schema = "sys",
     230             :                 .table = "types",
     231             :                 .column = "radix",
     232             :                 .fullname = "sys_types_radix",
     233             :                 .newid = 2013,
     234             :         },
     235             :         {
     236             :                 .schema = "sys",
     237             :                 .table = "types",
     238             :                 .column = "eclass",
     239             :                 .fullname = "sys_types_eclass",
     240             :                 .newid = 2014,
     241             :         },
     242             :         {
     243             :                 .schema = "sys",
     244             :                 .table = "types",
     245             :                 .column = "schema_id",
     246             :                 .fullname = "sys_types_schema_id",
     247             :                 .newid = 2015,
     248             :                 .hasids = true,
     249             :         },
     250             :         {
     251             :                 .schema = "sys",
     252             :                 .table = "functions",
     253             :                 .fullname = "D_sys_functions",
     254             :                 .newid = 2016,
     255             :         },
     256             :         {
     257             :                 .schema = "sys",
     258             :                 .table = "functions",
     259             :                 .column = "id",
     260             :                 .fullname = "sys_functions_id",
     261             :                 .newid = 2017,
     262             :         },
     263             :         {
     264             :                 .schema = "sys",
     265             :                 .table = "functions",
     266             :                 .column = "name",
     267             :                 .fullname = "sys_functions_name",
     268             :                 .newid = 2018,
     269             :         },
     270             :         {
     271             :                 .schema = "sys",
     272             :                 .table = "functions",
     273             :                 .column = "func",
     274             :                 .fullname = "sys_functions_func",
     275             :                 .newid = 2019,
     276             :         },
     277             :         {
     278             :                 .schema = "sys",
     279             :                 .table = "functions",
     280             :                 .column = "mod",
     281             :                 .fullname = "sys_functions_mod",
     282             :                 .newid = 2020,
     283             :         },
     284             :         {
     285             :                 .schema = "sys",
     286             :                 .table = "functions",
     287             :                 .column = "language",
     288             :                 .fullname = "sys_functions_language",
     289             :                 .newid = 2021,
     290             :         },
     291             :         {
     292             :                 .schema = "sys",
     293             :                 .table = "functions",
     294             :                 .column = "type",
     295             :                 .fullname = "sys_functions_type",
     296             :                 .newid = 2022,
     297             :         },
     298             :         {
     299             :                 .schema = "sys",
     300             :                 .table = "functions",
     301             :                 .column = "side_effect",
     302             :                 .fullname = "sys_functions_side_effect",
     303             :                 .newid = 2023,
     304             :         },
     305             :         {
     306             :                 .schema = "sys",
     307             :                 .table = "functions",
     308             :                 .column = "varres",
     309             :                 .fullname = "sys_functions_varres",
     310             :                 .newid = 2024,
     311             :         },
     312             :         {
     313             :                 .schema = "sys",
     314             :                 .table = "functions",
     315             :                 .column = "vararg",
     316             :                 .fullname = "sys_functions_vararg",
     317             :                 .newid = 2025,
     318             :         },
     319             :         {
     320             :                 .schema = "sys",
     321             :                 .table = "functions",
     322             :                 .column = "schema_id",
     323             :                 .fullname = "sys_functions_schema_id",
     324             :                 .newid = 2026,
     325             :                 .hasids = true,
     326             :         },
     327             :         {
     328             :                 .schema = "sys",
     329             :                 .table = "functions",
     330             :                 .column = "system",
     331             :                 .fullname = "sys_functions_system",
     332             :                 .newid = 2027,
     333             :         },
     334             :         {
     335             :                 .schema = "sys",
     336             :                 .table = "functions",
     337             :                 .column = "semantics",
     338             :                 .fullname = "sys_functions_semantics",
     339             :                 .newid = 2162,
     340             :         },
     341             :         {
     342             :                 .schema = "sys",
     343             :                 .table = "args",
     344             :                 .fullname = "D_sys_args",
     345             :                 .newid = 2028,
     346             :         },
     347             :         {
     348             :                 .schema = "sys",
     349             :                 .table = "args",
     350             :                 .column = "id",
     351             :                 .fullname = "sys_args_id",
     352             :                 .newid = 2029,
     353             :         },
     354             :         {
     355             :                 .schema = "sys",
     356             :                 .table = "args",
     357             :                 .column = "func_id",
     358             :                 .fullname = "sys_args_func_id",
     359             :                 .newid = 2030,
     360             :         },
     361             :         {
     362             :                 .schema = "sys",
     363             :                 .table = "args",
     364             :                 .column = "name",
     365             :                 .fullname = "sys_args_name",
     366             :                 .newid = 2031,
     367             :         },
     368             :         {
     369             :                 .schema = "sys",
     370             :                 .table = "args",
     371             :                 .column = "type",
     372             :                 .fullname = "sys_args_type",
     373             :                 .newid = 2032,
     374             :         },
     375             :         {
     376             :                 .schema = "sys",
     377             :                 .table = "args",
     378             :                 .column = "type_digits",
     379             :                 .fullname = "sys_args_type_digits",
     380             :                 .newid = 2033,
     381             :         },
     382             :         {
     383             :                 .schema = "sys",
     384             :                 .table = "args",
     385             :                 .column = "type_scale",
     386             :                 .fullname = "sys_args_type_scale",
     387             :                 .newid = 2034,
     388             :         },
     389             :         {
     390             :                 .schema = "sys",
     391             :                 .table = "args",
     392             :                 .column = "inout",
     393             :                 .fullname = "sys_args_inout",
     394             :                 .newid = 2035,
     395             :         },
     396             :         {
     397             :                 .schema = "sys",
     398             :                 .table = "args",
     399             :                 .column = "number",
     400             :                 .fullname = "sys_args_number",
     401             :                 .newid = 2036,
     402             :         },
     403             :         {
     404             :                 .schema = "sys",
     405             :                 .table = "sequences",
     406             :                 .fullname = "D_sys_sequences",
     407             :                 .newid = 2037,
     408             :         },
     409             :         {
     410             :                 .schema = "sys",
     411             :                 .table = "sequences",
     412             :                 .column = "id",
     413             :                 .fullname = "sys_sequences_id",
     414             :                 .newid = 2038,
     415             :         },
     416             :         {
     417             :                 .schema = "sys",
     418             :                 .table = "sequences",
     419             :                 .column = "schema_id",
     420             :                 .fullname = "sys_sequences_schema_id",
     421             :                 .newid = 2039,
     422             :                 .hasids = true,
     423             :         },
     424             :         {
     425             :                 .schema = "sys",
     426             :                 .table = "sequences",
     427             :                 .column = "name",
     428             :                 .fullname = "sys_sequences_name",
     429             :                 .newid = 2040,
     430             :         },
     431             :         {
     432             :                 .schema = "sys",
     433             :                 .table = "sequences",
     434             :                 .column = "start",
     435             :                 .fullname = "sys_sequences_start",
     436             :                 .newid = 2041,
     437             :         },
     438             :         {
     439             :                 .schema = "sys",
     440             :                 .table = "sequences",
     441             :                 .column = "minvalue",
     442             :                 .fullname = "sys_sequences_minvalue",
     443             :                 .newid = 2042,
     444             :         },
     445             :         {
     446             :                 .schema = "sys",
     447             :                 .table = "sequences",
     448             :                 .column = "maxvalue",
     449             :                 .fullname = "sys_sequences_maxvalue",
     450             :                 .newid = 2043,
     451             :         },
     452             :         {
     453             :                 .schema = "sys",
     454             :                 .table = "sequences",
     455             :                 .column = "increment",
     456             :                 .fullname = "sys_sequences_increment",
     457             :                 .newid = 2044,
     458             :         },
     459             :         {
     460             :                 .schema = "sys",
     461             :                 .table = "sequences",
     462             :                 .column = "cacheinc",
     463             :                 .fullname = "sys_sequences_cacheinc",
     464             :                 .newid = 2045,
     465             :         },
     466             :         {
     467             :                 .schema = "sys",
     468             :                 .table = "sequences",
     469             :                 .column = "cycle",
     470             :                 .fullname = "sys_sequences_cycle",
     471             :                 .newid = 2046,
     472             :         },
     473             :         {
     474             :                 .schema = "sys",
     475             :                 .table = "table_partitions",
     476             :                 .fullname = "D_sys_table_partitions",
     477             :                 .newid = 2047,
     478             :         },
     479             :         {
     480             :                 .schema = "sys",
     481             :                 .table = "table_partitions",
     482             :                 .column = "id",
     483             :                 .fullname = "sys_table_partitions_id",
     484             :                 .newid = 2048,
     485             :         },
     486             :         {
     487             :                 .schema = "sys",
     488             :                 .table = "table_partitions",
     489             :                 .column = "table_id",
     490             :                 .fullname = "sys_table_partitions_table_id",
     491             :                 .newid = 2049,
     492             :                 .hasids = true,
     493             :         },
     494             :         {
     495             :                 .schema = "sys",
     496             :                 .table = "table_partitions",
     497             :                 .column = "column_id",
     498             :                 .fullname = "sys_table_partitions_column_id",
     499             :                 .newid = 2050,
     500             :                 .hasids = true,
     501             :         },
     502             :         {
     503             :                 .schema = "sys",
     504             :                 .table = "table_partitions",
     505             :                 .column = "expression",
     506             :                 .fullname = "sys_table_partitions_expression",
     507             :                 .newid = 2051,
     508             :         },
     509             :         {
     510             :                 .schema = "sys",
     511             :                 .table = "table_partitions",
     512             :                 .column = "type",
     513             :                 .fullname = "sys_table_partitions_type",
     514             :                 .newid = 2052,
     515             :         },
     516             :         {
     517             :                 .schema = "sys",
     518             :                 .table = "range_partitions",
     519             :                 .fullname = "D_sys_range_partitions",
     520             :                 .newid = 2053,
     521             :         },
     522             :         {
     523             :                 .schema = "sys",
     524             :                 .table = "range_partitions",
     525             :                 .column = "table_id",
     526             :                 .fullname = "sys_range_partitions_table_id",
     527             :                 .newid = 2054,
     528             :                 .hasids = true,
     529             :         },
     530             :         {
     531             :                 .schema = "sys",
     532             :                 .table = "range_partitions",
     533             :                 .column = "partition_id",
     534             :                 .fullname = "sys_range_partitions_partition_id",
     535             :                 .newid = 2055,
     536             :         },
     537             :         {
     538             :                 .schema = "sys",
     539             :                 .table = "range_partitions",
     540             :                 .column = "minimum",
     541             :                 .fullname = "sys_range_partitions_minimum",
     542             :                 .newid = 2056,
     543             :         },
     544             :         {
     545             :                 .schema = "sys",
     546             :                 .table = "range_partitions",
     547             :                 .column = "maximum",
     548             :                 .fullname = "sys_range_partitions_maximum",
     549             :                 .newid = 2057,
     550             :         },
     551             :         {
     552             :                 .schema = "sys",
     553             :                 .table = "range_partitions",
     554             :                 .column = "with_nulls",
     555             :                 .fullname = "sys_range_partitions_with_nulls",
     556             :                 .newid = 2058,
     557             :         },
     558             :         {
     559             :                 .schema = "sys",
     560             :                 .table = "value_partitions",
     561             :                 .fullname = "D_sys_value_partitions",
     562             :                 .newid = 2059,
     563             :         },
     564             :         {
     565             :                 .schema = "sys",
     566             :                 .table = "value_partitions",
     567             :                 .column = "table_id",
     568             :                 .fullname = "sys_value_partitions_table_id",
     569             :                 .newid = 2060,
     570             :                 .hasids = true,
     571             :         },
     572             :         {
     573             :                 .schema = "sys",
     574             :                 .table = "value_partitions",
     575             :                 .column = "partition_id",
     576             :                 .fullname = "sys_value_partitions_partition_id",
     577             :                 .newid = 2061,
     578             :         },
     579             :         {
     580             :                 .schema = "sys",
     581             :                 .table = "value_partitions",
     582             :                 .column = "value",
     583             :                 .fullname = "sys_value_partitions_value",
     584             :                 .newid = 2062,
     585             :         },
     586             :         {
     587             :                 .schema = "sys",
     588             :                 .table = "dependencies",
     589             :                 .fullname = "D_sys_dependencies",
     590             :                 .newid = 2063,
     591             :         },
     592             :         {
     593             :                 .schema = "sys",
     594             :                 .table = "dependencies",
     595             :                 .column = "id",
     596             :                 .fullname = "sys_dependencies_id",
     597             :                 .newid = 2064,
     598             :                 .hasids = true,
     599             :         },
     600             :         {
     601             :                 .schema = "sys",
     602             :                 .table = "dependencies",
     603             :                 .column = "depend_id",
     604             :                 .fullname = "sys_dependencies_depend_id",
     605             :                 .newid = 2065,
     606             :                 .hasids = true,
     607             :         },
     608             :         {
     609             :                 .schema = "sys",
     610             :                 .table = "dependencies",
     611             :                 .column = "depend_type",
     612             :                 .fullname = "sys_dependencies_depend_type",
     613             :                 .newid = 2066,
     614             :         },
     615             :         {
     616             :                 .schema = "sys",
     617             :                 .table = "_tables",
     618             :                 .fullname = "D_sys__tables",
     619             :                 .newid = 2067,
     620             :         },
     621             :         {
     622             :                 .schema = "sys",
     623             :                 .table = "_tables",
     624             :                 .column = "id",
     625             :                 .fullname = "sys__tables_id",
     626             :                 .newid = 2068,
     627             :                 .hasids = true,
     628             :         },
     629             :         {
     630             :                 .schema = "sys",
     631             :                 .table = "_tables",
     632             :                 .column = "name",
     633             :                 .fullname = "sys__tables_name",
     634             :                 .newid = 2069,
     635             :         },
     636             :         {
     637             :                 .schema = "sys",
     638             :                 .table = "_tables",
     639             :                 .column = "schema_id",
     640             :                 .fullname = "sys__tables_schema_id",
     641             :                 .newid = 2070,
     642             :                 .hasids = true,
     643             :         },
     644             :         {
     645             :                 .schema = "sys",
     646             :                 .table = "_tables",
     647             :                 .column = "query",
     648             :                 .fullname = "sys__tables_query",
     649             :                 .newid = 2071,
     650             :         },
     651             :         {
     652             :                 .schema = "sys",
     653             :                 .table = "_tables",
     654             :                 .column = "type",
     655             :                 .fullname = "sys__tables_type",
     656             :                 .newid = 2072,
     657             :         },
     658             :         {
     659             :                 .schema = "sys",
     660             :                 .table = "_tables",
     661             :                 .column = "system",
     662             :                 .fullname = "sys__tables_system",
     663             :                 .newid = 2073,
     664             :         },
     665             :         {
     666             :                 .schema = "sys",
     667             :                 .table = "_tables",
     668             :                 .column = "commit_action",
     669             :                 .fullname = "sys__tables_commit_action",
     670             :                 .newid = 2074,
     671             :         },
     672             :         {
     673             :                 .schema = "sys",
     674             :                 .table = "_tables",
     675             :                 .column = "access",
     676             :                 .fullname = "sys__tables_access",
     677             :                 .newid = 2075,
     678             :         },
     679             :         {
     680             :                 .schema = "sys",
     681             :                 .table = "_columns",
     682             :                 .fullname = "D_sys__columns",
     683             :                 .newid = 2076,
     684             :         },
     685             :         {
     686             :                 .schema = "sys",
     687             :                 .table = "_columns",
     688             :                 .column = "id",
     689             :                 .fullname = "sys__columns_id",
     690             :                 .newid = 2077,
     691             :                 .hasids = true,
     692             :         },
     693             :         {
     694             :                 .schema = "sys",
     695             :                 .table = "_columns",
     696             :                 .column = "name",
     697             :                 .fullname = "sys__columns_name",
     698             :                 .newid = 2078,
     699             :         },
     700             :         {
     701             :                 .schema = "sys",
     702             :                 .table = "_columns",
     703             :                 .column = "type",
     704             :                 .fullname = "sys__columns_type",
     705             :                 .newid = 2079,
     706             :         },
     707             :         {
     708             :                 .schema = "sys",
     709             :                 .table = "_columns",
     710             :                 .column = "type_digits",
     711             :                 .fullname = "sys__columns_type_digits",
     712             :                 .newid = 2080,
     713             :         },
     714             :         {
     715             :                 .schema = "sys",
     716             :                 .table = "_columns",
     717             :                 .column = "type_scale",
     718             :                 .fullname = "sys__columns_type_scale",
     719             :                 .newid = 2081,
     720             :         },
     721             :         {
     722             :                 .schema = "sys",
     723             :                 .table = "_columns",
     724             :                 .column = "table_id",
     725             :                 .fullname = "sys__columns_table_id",
     726             :                 .newid = 2082,
     727             :                 .hasids = true,
     728             :         },
     729             :         {
     730             :                 .schema = "sys",
     731             :                 .table = "_columns",
     732             :                 .column = "default",
     733             :                 .fullname = "sys__columns_default",
     734             :                 .newid = 2083,
     735             :         },
     736             :         {
     737             :                 .schema = "sys",
     738             :                 .table = "_columns",
     739             :                 .column = "null",
     740             :                 .fullname = "sys__columns_null",
     741             :                 .newid = 2084,
     742             :         },
     743             :         {
     744             :                 .schema = "sys",
     745             :                 .table = "_columns",
     746             :                 .column = "number",
     747             :                 .fullname = "sys__columns_number",
     748             :                 .newid = 2085,
     749             :         },
     750             :         {
     751             :                 .schema = "sys",
     752             :                 .table = "_columns",
     753             :                 .column = "storage",
     754             :                 .fullname = "sys__columns_storage",
     755             :                 .newid = 2086,
     756             :         },
     757             :         {
     758             :                 .schema = "sys",
     759             :                 .table = "keys",
     760             :                 .fullname = "D_sys_keys",
     761             :                 .newid = 2087,
     762             :         },
     763             :         {
     764             :                 .schema = "sys",
     765             :                 .table = "keys",
     766             :                 .column = "id",
     767             :                 .fullname = "sys_keys_id",
     768             :                 .newid = 2088,
     769             :         },
     770             :         {
     771             :                 .schema = "sys",
     772             :                 .table = "keys",
     773             :                 .column = "table_id",
     774             :                 .fullname = "sys_keys_table_id",
     775             :                 .newid = 2089,
     776             :                 .hasids = true,
     777             :         },
     778             :         {
     779             :                 .schema = "sys",
     780             :                 .table = "keys",
     781             :                 .column = "type",
     782             :                 .fullname = "sys_keys_type",
     783             :                 .newid = 2090,
     784             :         },
     785             :         {
     786             :                 .schema = "sys",
     787             :                 .table = "keys",
     788             :                 .column = "name",
     789             :                 .fullname = "sys_keys_name",
     790             :                 .newid = 2091,
     791             :         },
     792             :         {
     793             :                 .schema = "sys",
     794             :                 .table = "keys",
     795             :                 .column = "rkey",
     796             :                 .fullname = "sys_keys_rkey",
     797             :                 .newid = 2092,
     798             :         },
     799             :         {
     800             :                 .schema = "sys",
     801             :                 .table = "keys",
     802             :                 .column = "action",
     803             :                 .fullname = "sys_keys_action",
     804             :                 .newid = 2093,
     805             :         },
     806             :         {
     807             :                 .schema = "sys",
     808             :                 .table = "idxs",
     809             :                 .fullname = "D_sys_idxs",
     810             :                 .newid = 2094,
     811             :         },
     812             :         {
     813             :                 .schema = "sys",
     814             :                 .table = "idxs",
     815             :                 .column = "id",
     816             :                 .fullname = "sys_idxs_id",
     817             :                 .newid = 2095,
     818             :         },
     819             :         {
     820             :                 .schema = "sys",
     821             :                 .table = "idxs",
     822             :                 .column = "table_id",
     823             :                 .fullname = "sys_idxs_table_id",
     824             :                 .newid = 2096,
     825             :                 .hasids = true,
     826             :         },
     827             :         {
     828             :                 .schema = "sys",
     829             :                 .table = "idxs",
     830             :                 .column = "type",
     831             :                 .fullname = "sys_idxs_type",
     832             :                 .newid = 2097,
     833             :         },
     834             :         {
     835             :                 .schema = "sys",
     836             :                 .table = "idxs",
     837             :                 .column = "name",
     838             :                 .fullname = "sys_idxs_name",
     839             :                 .newid = 2098,
     840             :         },
     841             :         {
     842             :                 .schema = "sys",
     843             :                 .table = "triggers",
     844             :                 .fullname = "D_sys_triggers",
     845             :                 .newid = 2099,
     846             :         },
     847             :         {
     848             :                 .schema = "sys",
     849             :                 .table = "triggers",
     850             :                 .column = "id",
     851             :                 .fullname = "sys_triggers_id",
     852             :                 .newid = 2100,
     853             :         },
     854             :         {
     855             :                 .schema = "sys",
     856             :                 .table = "triggers",
     857             :                 .column = "name",
     858             :                 .fullname = "sys_triggers_name",
     859             :                 .newid = 2101,
     860             :         },
     861             :         {
     862             :                 .schema = "sys",
     863             :                 .table = "triggers",
     864             :                 .column = "table_id",
     865             :                 .fullname = "sys_triggers_table_id",
     866             :                 .newid = 2102,
     867             :                 .hasids = true,
     868             :         },
     869             :         {
     870             :                 .schema = "sys",
     871             :                 .table = "triggers",
     872             :                 .column = "time",
     873             :                 .fullname = "sys_triggers_time",
     874             :                 .newid = 2103,
     875             :         },
     876             :         {
     877             :                 .schema = "sys",
     878             :                 .table = "triggers",
     879             :                 .column = "orientation",
     880             :                 .fullname = "sys_triggers_orientation",
     881             :                 .newid = 2104,
     882             :         },
     883             :         {
     884             :                 .schema = "sys",
     885             :                 .table = "triggers",
     886             :                 .column = "event",
     887             :                 .fullname = "sys_triggers_event",
     888             :                 .newid = 2105,
     889             :         },
     890             :         {
     891             :                 .schema = "sys",
     892             :                 .table = "triggers",
     893             :                 .column = "old_name",
     894             :                 .fullname = "sys_triggers_old_name",
     895             :                 .newid = 2106,
     896             :         },
     897             :         {
     898             :                 .schema = "sys",
     899             :                 .table = "triggers",
     900             :                 .column = "new_name",
     901             :                 .fullname = "sys_triggers_new_name",
     902             :                 .newid = 2107,
     903             :         },
     904             :         {
     905             :                 .schema = "sys",
     906             :                 .table = "triggers",
     907             :                 .column = "condition",
     908             :                 .fullname = "sys_triggers_condition",
     909             :                 .newid = 2108,
     910             :         },
     911             :         {
     912             :                 .schema = "sys",
     913             :                 .table = "triggers",
     914             :                 .column = "statement",
     915             :                 .fullname = "sys_triggers_statement",
     916             :                 .newid = 2109,
     917             :         },
     918             :         {
     919             :                 .schema = "sys",
     920             :                 .table = "objects",
     921             :                 .fullname = "D_sys_objects",
     922             :                 .newid = 2110,
     923             :         },
     924             :         {
     925             :                 .schema = "sys",
     926             :                 .table = "objects",
     927             :                 .column = "id",
     928             :                 .fullname = "sys_objects_id",
     929             :                 .newid = 2111,
     930             :         },
     931             :         {
     932             :                 .schema = "sys",
     933             :                 .table = "objects",
     934             :                 .column = "name",
     935             :                 .fullname = "sys_objects_name",
     936             :                 .newid = 2112,
     937             :         },
     938             :         {
     939             :                 .schema = "sys",
     940             :                 .table = "objects",
     941             :                 .column = "nr",
     942             :                 .fullname = "sys_objects_nr",
     943             :                 .newid = 2113,
     944             :                 .hasids = true,
     945             :         },
     946             :         {
     947             :                 .schema = "sys",
     948             :                 .table = "objects",
     949             :                 .column = "sub",
     950             :                 .fullname = "sys_objects_sub",
     951             :                 .newid = 2163,
     952             :                 .hasids = true,
     953             :         },
     954             :         {
     955             :                 .schema = "tmp",
     956             :                 .newid = 2114,
     957             :         },
     958             :         {
     959             :                 .schema = "tmp",
     960             :                 .table = "_tables",
     961             :                 .fullname = "D_tmp__tables",
     962             :                 .newid = 2115,
     963             :         },
     964             :         {
     965             :                 .schema = "tmp",
     966             :                 .table = "_tables",
     967             :                 .column = "id",
     968             :                 .fullname = "tmp__tables_id",
     969             :                 .newid = 2116,
     970             :         },
     971             :         {
     972             :                 .schema = "tmp",
     973             :                 .table = "_tables",
     974             :                 .column = "name",
     975             :                 .fullname = "tmp__tables_name",
     976             :                 .newid = 2117,
     977             :         },
     978             :         {
     979             :                 .schema = "tmp",
     980             :                 .table = "_tables",
     981             :                 .column = "schema_id",
     982             :                 .fullname = "tmp__tables_schema_id",
     983             :                 .newid = 2118,
     984             :         },
     985             :         {
     986             :                 .schema = "tmp",
     987             :                 .table = "_tables",
     988             :                 .column = "query",
     989             :                 .fullname = "tmp__tables_query",
     990             :                 .newid = 2119,
     991             :         },
     992             :         {
     993             :                 .schema = "tmp",
     994             :                 .table = "_tables",
     995             :                 .column = "type",
     996             :                 .fullname = "tmp__tables_type",
     997             :                 .newid = 2120,
     998             :         },
     999             :         {
    1000             :                 .schema = "tmp",
    1001             :                 .table = "_tables",
    1002             :                 .column = "system",
    1003             :                 .fullname = "tmp__tables_system",
    1004             :                 .newid = 2121,
    1005             :         },
    1006             :         {
    1007             :                 .schema = "tmp",
    1008             :                 .table = "_tables",
    1009             :                 .column = "commit_action",
    1010             :                 .fullname = "tmp__tables_commit_action",
    1011             :                 .newid = 2122,
    1012             :         },
    1013             :         {
    1014             :                 .schema = "tmp",
    1015             :                 .table = "_tables",
    1016             :                 .column = "access",
    1017             :                 .fullname = "tmp__tables_access",
    1018             :                 .newid = 2123,
    1019             :         },
    1020             :         {
    1021             :                 .schema = "tmp",
    1022             :                 .table = "_columns",
    1023             :                 .fullname = "D_tmp__columns",
    1024             :                 .newid = 2124,
    1025             :         },
    1026             :         {
    1027             :                 .schema = "tmp",
    1028             :                 .table = "_columns",
    1029             :                 .column = "id",
    1030             :                 .fullname = "tmp__columns_id",
    1031             :                 .newid = 2125,
    1032             :         },
    1033             :         {
    1034             :                 .schema = "tmp",
    1035             :                 .table = "_columns",
    1036             :                 .column = "name",
    1037             :                 .fullname = "tmp__columns_name",
    1038             :                 .newid = 2126,
    1039             :         },
    1040             :         {
    1041             :                 .schema = "tmp",
    1042             :                 .table = "_columns",
    1043             :                 .column = "type",
    1044             :                 .fullname = "tmp__columns_type",
    1045             :                 .newid = 2127,
    1046             :         },
    1047             :         {
    1048             :                 .schema = "tmp",
    1049             :                 .table = "_columns",
    1050             :                 .column = "type_digits",
    1051             :                 .fullname = "tmp__columns_type_digits",
    1052             :                 .newid = 2128,
    1053             :         },
    1054             :         {
    1055             :                 .schema = "tmp",
    1056             :                 .table = "_columns",
    1057             :                 .column = "type_scale",
    1058             :                 .fullname = "tmp__columns_type_scale",
    1059             :                 .newid = 2129,
    1060             :         },
    1061             :         {
    1062             :                 .schema = "tmp",
    1063             :                 .table = "_columns",
    1064             :                 .column = "table_id",
    1065             :                 .fullname = "tmp__columns_table_id",
    1066             :                 .newid = 2130,
    1067             :         },
    1068             :         {
    1069             :                 .schema = "tmp",
    1070             :                 .table = "_columns",
    1071             :                 .column = "default",
    1072             :                 .fullname = "tmp__columns_default",
    1073             :                 .newid = 2131,
    1074             :         },
    1075             :         {
    1076             :                 .schema = "tmp",
    1077             :                 .table = "_columns",
    1078             :                 .column = "null",
    1079             :                 .fullname = "tmp__columns_null",
    1080             :                 .newid = 2132,
    1081             :         },
    1082             :         {
    1083             :                 .schema = "tmp",
    1084             :                 .table = "_columns",
    1085             :                 .column = "number",
    1086             :                 .fullname = "tmp__columns_number",
    1087             :                 .newid = 2133,
    1088             :         },
    1089             :         {
    1090             :                 .schema = "tmp",
    1091             :                 .table = "_columns",
    1092             :                 .column = "storage",
    1093             :                 .fullname = "tmp__columns_storage",
    1094             :                 .newid = 2134,
    1095             :         },
    1096             :         {
    1097             :                 .schema = "tmp",
    1098             :                 .table = "keys",
    1099             :                 .fullname = "D_tmp_keys",
    1100             :                 .newid = 2135,
    1101             :         },
    1102             :         {
    1103             :                 .schema = "tmp",
    1104             :                 .table = "keys",
    1105             :                 .column = "id",
    1106             :                 .fullname = "tmp_keys_id",
    1107             :                 .newid = 2136,
    1108             :         },
    1109             :         {
    1110             :                 .schema = "tmp",
    1111             :                 .table = "keys",
    1112             :                 .column = "table_id",
    1113             :                 .fullname = "tmp_keys_table_id",
    1114             :                 .newid = 2137,
    1115             :         },
    1116             :         {
    1117             :                 .schema = "tmp",
    1118             :                 .table = "keys",
    1119             :                 .column = "type",
    1120             :                 .fullname = "tmp_keys_type",
    1121             :                 .newid = 2138,
    1122             :         },
    1123             :         {
    1124             :                 .schema = "tmp",
    1125             :                 .table = "keys",
    1126             :                 .column = "name",
    1127             :                 .fullname = "tmp_keys_name",
    1128             :                 .newid = 2139,
    1129             :         },
    1130             :         {
    1131             :                 .schema = "tmp",
    1132             :                 .table = "keys",
    1133             :                 .column = "rkey",
    1134             :                 .fullname = "tmp_keys_rkey",
    1135             :                 .newid = 2140,
    1136             :         },
    1137             :         {
    1138             :                 .schema = "tmp",
    1139             :                 .table = "keys",
    1140             :                 .column = "action",
    1141             :                 .fullname = "tmp_keys_action",
    1142             :                 .newid = 2141,
    1143             :         },
    1144             :         {
    1145             :                 .schema = "tmp",
    1146             :                 .table = "idxs",
    1147             :                 .fullname = "D_tmp_idxs",
    1148             :                 .newid = 2142,
    1149             :         },
    1150             :         {
    1151             :                 .schema = "tmp",
    1152             :                 .table = "idxs",
    1153             :                 .column = "id",
    1154             :                 .fullname = "tmp_idxs_id",
    1155             :                 .newid = 2143,
    1156             :         },
    1157             :         {
    1158             :                 .schema = "tmp",
    1159             :                 .table = "idxs",
    1160             :                 .column = "table_id",
    1161             :                 .fullname = "tmp_idxs_table_id",
    1162             :                 .newid = 2144,
    1163             :         },
    1164             :         {
    1165             :                 .schema = "tmp",
    1166             :                 .table = "idxs",
    1167             :                 .column = "type",
    1168             :                 .fullname = "tmp_idxs_type",
    1169             :                 .newid = 2145,
    1170             :         },
    1171             :         {
    1172             :                 .schema = "tmp",
    1173             :                 .table = "idxs",
    1174             :                 .column = "name",
    1175             :                 .fullname = "tmp_idxs_name",
    1176             :                 .newid = 2146,
    1177             :         },
    1178             :         {
    1179             :                 .schema = "tmp",
    1180             :                 .table = "triggers",
    1181             :                 .fullname = "D_tmp_triggers",
    1182             :                 .newid = 2147,
    1183             :         },
    1184             :         {
    1185             :                 .schema = "tmp",
    1186             :                 .table = "triggers",
    1187             :                 .column = "id",
    1188             :                 .fullname = "tmp_triggers_id",
    1189             :                 .newid = 2148,
    1190             :         },
    1191             :         {
    1192             :                 .schema = "tmp",
    1193             :                 .table = "triggers",
    1194             :                 .column = "name",
    1195             :                 .fullname = "tmp_triggers_name",
    1196             :                 .newid = 2149,
    1197             :         },
    1198             :         {
    1199             :                 .schema = "tmp",
    1200             :                 .table = "triggers",
    1201             :                 .column = "table_id",
    1202             :                 .fullname = "tmp_triggers_table_id",
    1203             :                 .newid = 2150,
    1204             :         },
    1205             :         {
    1206             :                 .schema = "tmp",
    1207             :                 .table = "triggers",
    1208             :                 .column = "time",
    1209             :                 .fullname = "tmp_triggers_time",
    1210             :                 .newid = 2151,
    1211             :         },
    1212             :         {
    1213             :                 .schema = "tmp",
    1214             :                 .table = "triggers",
    1215             :                 .column = "orientation",
    1216             :                 .fullname = "tmp_triggers_orientation",
    1217             :                 .newid = 2152,
    1218             :         },
    1219             :         {
    1220             :                 .schema = "tmp",
    1221             :                 .table = "triggers",
    1222             :                 .column = "event",
    1223             :                 .fullname = "tmp_triggers_event",
    1224             :                 .newid = 2153,
    1225             :         },
    1226             :         {
    1227             :                 .schema = "tmp",
    1228             :                 .table = "triggers",
    1229             :                 .column = "old_name",
    1230             :                 .fullname = "tmp_triggers_old_name",
    1231             :                 .newid = 2154,
    1232             :         },
    1233             :         {
    1234             :                 .schema = "tmp",
    1235             :                 .table = "triggers",
    1236             :                 .column = "new_name",
    1237             :                 .fullname = "tmp_triggers_new_name",
    1238             :                 .newid = 2155,
    1239             :         },
    1240             :         {
    1241             :                 .schema = "tmp",
    1242             :                 .table = "triggers",
    1243             :                 .column = "condition",
    1244             :                 .fullname = "tmp_triggers_condition",
    1245             :                 .newid = 2156,
    1246             :         },
    1247             :         {
    1248             :                 .schema = "tmp",
    1249             :                 .table = "triggers",
    1250             :                 .column = "statement",
    1251             :                 .fullname = "tmp_triggers_statement",
    1252             :                 .newid = 2157,
    1253             :         },
    1254             :         {
    1255             :                 .schema = "tmp",
    1256             :                 .table = "objects",
    1257             :                 .fullname = "D_tmp_objects",
    1258             :                 .newid = 2158,
    1259             :         },
    1260             :         {
    1261             :                 .schema = "tmp",
    1262             :                 .table = "objects",
    1263             :                 .column = "id",
    1264             :                 .fullname = "tmp_objects_id",
    1265             :                 .newid = 2159,
    1266             :         },
    1267             :         {
    1268             :                 .schema = "tmp",
    1269             :                 .table = "objects",
    1270             :                 .column = "name",
    1271             :                 .fullname = "tmp_objects_name",
    1272             :                 .newid = 2160,
    1273             :         },
    1274             :         {
    1275             :                 .schema = "tmp",
    1276             :                 .table = "objects",
    1277             :                 .column = "nr",
    1278             :                 .fullname = "tmp_objects_nr",
    1279             :                 .newid = 2161,
    1280             :         },
    1281             :         {
    1282             :                 .schema = "tmp",
    1283             :                 .table = "objects",
    1284             :                 .column = "sub",
    1285             :                 .fullname = "tmp_objects_sub",
    1286             :                 .newid = 2164,
    1287             :         },
    1288             :         {0}
    1289             : };
    1290             : 
    1291             : /* more system tables with schema/table/column ids that need to be remapped */
    1292             : struct mapids {
    1293             :         // const char *schema;                  /* always "sys" */
    1294             :         const char *table;
    1295             :         const char *column;
    1296             : } mapids[] = {
    1297             :         {
    1298             :                 .table = "comments",
    1299             :                 .column = "id",
    1300             :         },
    1301             :         {
    1302             :                 .table = "db_user_info",
    1303             :                 .column = "default_schema",
    1304             :         },
    1305             :         {
    1306             :                 .table = "privileges",
    1307             :                 .column = "obj_id",
    1308             :         },
    1309             :         {
    1310             :                 .table = "statistics",
    1311             :                 .column = "column_id",
    1312             :         },
    1313             :         {0}
    1314             : };
    1315             : 
    1316             : static gdk_return
    1317           8 : upgrade(old_logger *lg)
    1318             : {
    1319             :         gdk_return rc = GDK_FAIL;
    1320             :         struct bats {
    1321             :                 BAT *nmbat;
    1322             :                 BAT *idbat;
    1323             :                 BAT *parbat;
    1324             :                 BAT *cands;
    1325             :         } bats[3];
    1326           8 :         BAT *mapold = COLnew(0, TYPE_int, 256, TRANSIENT);
    1327           8 :         BAT *mapnew = COLnew(0, TYPE_int, 256, TRANSIENT);
    1328             : 
    1329           8 :         bats[0].nmbat = temp_descriptor(old_logger_find_bat(lg, "sys_schemas_name", 0, 0));
    1330           8 :         bats[0].idbat = temp_descriptor(old_logger_find_bat(lg, "sys_schemas_id", 0, 0));
    1331           8 :         bats[0].parbat = NULL;
    1332           8 :         bats[0].cands = temp_descriptor(old_logger_find_bat(lg, "D_sys_schemas", 0, 0));
    1333           8 :         bats[1].nmbat = temp_descriptor(old_logger_find_bat(lg, "sys__tables_name", 0, 0));
    1334           8 :         bats[1].idbat = temp_descriptor(old_logger_find_bat(lg, "sys__tables_id", 0, 0));
    1335           8 :         bats[1].parbat = temp_descriptor(old_logger_find_bat(lg, "sys__tables_schema_id", 0, 0));
    1336           8 :         bats[1].cands = temp_descriptor(old_logger_find_bat(lg, "D_sys__tables", 0, 0));
    1337           8 :         bats[2].nmbat = temp_descriptor(old_logger_find_bat(lg, "sys__columns_name", 0, 0));
    1338           8 :         bats[2].idbat = temp_descriptor(old_logger_find_bat(lg, "sys__columns_id", 0, 0));
    1339           8 :         bats[2].parbat = temp_descriptor(old_logger_find_bat(lg, "sys__columns_table_id", 0, 0));
    1340           8 :         bats[2].cands = temp_descriptor(old_logger_find_bat(lg, "D_sys__columns", 0, 0));
    1341           8 :         if (mapold == NULL || mapnew == NULL)
    1342           0 :                 goto bailout;
    1343          32 :         for (int i = 0; i < 3; i++) {
    1344          24 :                 if (bats[i].nmbat == NULL || bats[i].idbat == NULL || bats[i].cands == NULL)
    1345           0 :                         goto bailout;
    1346          24 :                 if (i > 0 && bats[i].parbat == NULL)
    1347           0 :                         goto bailout;
    1348             :                 /* create a candidate list from the deleted rows bat */
    1349          24 :                 if (BATcount(bats[i].cands) == 0) {
    1350             :                         /* no deleted rows -> no candidate list */
    1351          16 :                         bat_destroy(bats[i].cands);
    1352          16 :                         bats[i].cands = NULL;
    1353             :                 } else {
    1354             :                         BAT *b;
    1355           8 :                         if ((rc = BATsort(&b, NULL, NULL, bats[i].cands, NULL, NULL, false, false, false)) != GDK_SUCCEED)
    1356           0 :                                 goto bailout;
    1357             :                         rc = GDK_FAIL;
    1358           8 :                         bat_destroy(bats[i].cands);
    1359           8 :                         bats[i].cands = BATnegcands(BATcount(bats[i].nmbat), b);
    1360           8 :                         bat_destroy(b);
    1361           8 :                         if (bats[i].cands == NULL) {
    1362           0 :                                 goto bailout;
    1363             :                         }
    1364             :                 }
    1365             :         }
    1366             : 
    1367             :         /* figure out mapping from old IDs to new stable IDs, result in two
    1368             :          * aligned BATs, mapold and mapnew */
    1369             :         int schid, tabid, parid;
    1370           8 :         schid = tabid = parid = 0;      /* restrict search to parent object */
    1371        1328 :         for (int i = 0; tables[i].schema != NULL; i++) {
    1372             :                 int lookup;                             /* which system table to look the name up in */
    1373             :                 const char *name;               /* the name to look up */
    1374        1320 :                 if (tables[i].table == NULL) {
    1375             :                         /* it's a schema */
    1376             :                         name = tables[i].schema;
    1377             :                         lookup = 0;
    1378          16 :                         parid = 0;                      /* no parent object */
    1379        1304 :                 } else if (tables[i].column == NULL) {
    1380             :                         /* it's a table */
    1381             :                         name = tables[i].table;
    1382             :                         lookup = 1;
    1383         168 :                         parid = schid;          /* parent object is last schema */
    1384             :                 } else {
    1385             :                         /* it's a column */
    1386             :                         name = tables[i].column;
    1387             :                         lookup = 2;
    1388        1136 :                         parid = tabid;          /* parent object is last table */
    1389             :                 }
    1390             :                 /* restrict search to non-deleted rows */
    1391        1320 :                 BAT *cand = bats[lookup].cands;
    1392        1320 :                 if (bats[lookup].parbat != NULL) {
    1393             :                         /* further restrict search to parent object */
    1394        1304 :                         cand = BATselect(bats[lookup].parbat, cand, &parid, NULL, true, true, false);
    1395        1304 :                         if (cand == NULL)
    1396           0 :                                 goto bailout;
    1397             :                 }
    1398             :                 /* look for name, should be one (or maybe zero) result */
    1399        1320 :                 BAT *b = BATselect(bats[lookup].nmbat, cand, name, NULL, true, true, false);
    1400        1320 :                 if (cand != bats[lookup].cands)
    1401        1304 :                         bat_destroy(cand);
    1402        1320 :                 if (b == NULL)
    1403           0 :                         goto bailout;
    1404        1320 :                 if (BATcount(b) > 0) {
    1405        1304 :                         int oldid = ((int *) bats[lookup].idbat->theap->base)[BUNtoid(b, 0) - bats[lookup].nmbat->hseqbase];
    1406        2430 :                         if (oldid != tables[i].newid &&
    1407        2252 :                                 ((rc = BUNappend(mapold, &oldid, false)) != GDK_SUCCEED ||
    1408        1126 :                                  (rc = BUNappend(mapnew, &tables[i].newid, false)) != GDK_SUCCEED)) {
    1409           0 :                                 bat_destroy(b);
    1410           0 :                                 goto bailout;
    1411             :                         }
    1412             :                         rc = GDK_FAIL;
    1413        1304 :                         if (tables[i].table == NULL)
    1414          16 :                                 schid = oldid;
    1415        1288 :                         else if (tables[i].column == NULL)
    1416         168 :                                 tabid = oldid;
    1417             :                 }
    1418        1320 :                 bat_destroy(b);
    1419             :         }
    1420             : 
    1421           8 :         if (BATcount(mapold) == 0) {
    1422             :                 /* skip unnecessary work if there is no need for mapping */
    1423           0 :                 bat_destroy(mapold);
    1424           0 :                 bat_destroy(mapnew);
    1425             :                 mapold = NULL;
    1426             :                 mapnew = NULL;
    1427             :         }
    1428             : 
    1429             :         /* do the mapping in the system tables: all columns with the .hasids
    1430             :          * flag set may contain IDs that have to be mapped; also add all
    1431             :          * system tables to the new catalog bats and add the new ones to the
    1432             :          * lg->add bat and the old ones that were replaced to the lg->del bat */
    1433             :         const char *delname;
    1434             :         delname = NULL;
    1435             :         int delidx;
    1436             :         delidx = -1;
    1437        1328 :         for (int i = 0; tables[i].schema != NULL; i++) {
    1438        1320 :                 if (tables[i].fullname == NULL) /* schema */
    1439          16 :                         continue;
    1440        1304 :                 if (tables[i].column == NULL) { /* table */
    1441             :                         delname = tables[i].fullname;
    1442             :                         delidx = i;
    1443         168 :                         continue;
    1444             :                 }
    1445        1136 :                 BAT *b = temp_descriptor(old_logger_find_bat(lg, tables[i].fullname, 0, 0));
    1446        1136 :                 if (b == NULL)
    1447         344 :                         continue;
    1448         792 :                 if (delidx >= 0) {
    1449         120 :                         BAT *d = temp_descriptor(old_logger_find_bat(lg, delname, 0, 0));
    1450         120 :                         BAT *m = BATconstant(0, TYPE_msk, &(msk){false}, BATcount(b), PERSISTENT);
    1451         120 :                         if (m == NULL) {
    1452           0 :                                 bat_destroy(d);
    1453           0 :                                 bat_destroy(m);
    1454           0 :                                 goto bailout;
    1455             :                         }
    1456         120 :                         if (d != NULL) {
    1457         108 :                                 const oid *dels = (const oid *) Tloc(d, 0);
    1458       59558 :                                 for (BUN q = BUNlast(d), p = 0; p < q; p++)
    1459       59450 :                                         mskSetVal(m, (BUN) dels[p], true);
    1460         108 :                                 BBPretain(d->batCacheid);
    1461             :                         }
    1462         240 :                         if ((rc = BUNappend(lg->add, &m->batCacheid, false)) != GDK_SUCCEED ||
    1463         240 :                                 (rc = BUNappend(lg->lg->catalog_bid, &m->batCacheid, false)) != GDK_SUCCEED ||
    1464         240 :                                 (rc = BUNappend(lg->lg->catalog_id, &tables[delidx].newid, false)) != GDK_SUCCEED ||
    1465         108 :                                 (d != NULL &&
    1466         108 :                                  (rc = BUNappend(lg->del, &d->batCacheid, false)) != GDK_SUCCEED)) {
    1467           0 :                                 bat_destroy(d);
    1468           0 :                                 bat_destroy(m);
    1469           0 :                                 goto bailout;
    1470             :                         }
    1471             :                         rc = GDK_FAIL;
    1472         120 :                         BBPretain(m->batCacheid);
    1473         120 :                         BBPretain(m->batCacheid);
    1474         120 :                         bat_destroy(d);
    1475         120 :                         bat_destroy(m);
    1476             :                         delidx = -1;
    1477             :                 }
    1478         792 :                 if (tables[i].hasids && mapold) {
    1479             :                         BAT *b1, *b2;
    1480         144 :                         BAT *cands = temp_descriptor(old_logger_find_bat(lg, delname, 0, 0));
    1481         144 :                         if (cands) {
    1482         128 :                                 if (BATcount(cands) == 0) {
    1483          96 :                                         bat_destroy(cands);
    1484             :                                         cands = NULL;
    1485             :                                 } else {
    1486          32 :                                         rc = BATsort(&b1, NULL, NULL, cands, NULL, NULL, false, false, false);
    1487          32 :                                         bat_destroy(cands);
    1488          32 :                                         if (rc != GDK_SUCCEED) {
    1489           0 :                                                 bat_destroy(b);
    1490           0 :                                                 goto bailout;
    1491             :                                         }
    1492             :                                         rc = GDK_FAIL;
    1493          32 :                                         cands = BATnegcands(BATcount(b), b1);
    1494          32 :                                         bat_destroy(b1);
    1495          32 :                                         if (cands == NULL) {
    1496           0 :                                                 bat_destroy(b);
    1497           0 :                                                 goto bailout;
    1498             :                                         }
    1499             :                                 }
    1500             :                         }
    1501         144 :                         rc = BATjoin(&b1, &b2, b, mapold, cands, NULL, false, BATcount(mapold));
    1502         144 :                         bat_destroy(cands);
    1503         144 :                         if (rc != GDK_SUCCEED) {
    1504           0 :                                 bat_destroy(b);
    1505           0 :                                 goto bailout;
    1506             :                         }
    1507             :                         rc = GDK_FAIL;
    1508         144 :                         if (BATcount(b1) == 0) {
    1509          92 :                                 bat_destroy(b1);
    1510          92 :                                 bat_destroy(b2);
    1511             :                         } else {
    1512             :                                 BAT *orig = b;
    1513          52 :                                 b = COLcopy(orig, orig->ttype, true, PERSISTENT);
    1514          52 :                                 if (b == NULL) {
    1515           0 :                                         bat_destroy(orig);
    1516           0 :                                         bat_destroy(b1);
    1517           0 :                                         bat_destroy(b2);
    1518           0 :                                         goto bailout;
    1519             :                                 }
    1520             :                                 BAT *b3;
    1521          52 :                                 b3 = BATproject(b2, mapnew);
    1522          52 :                                 bat_destroy(b2);
    1523          52 :                                 rc = BATreplace(b, b1, b3, false);
    1524          52 :                                 bat_destroy(b1);
    1525          52 :                                 bat_destroy(b3);
    1526          52 :                                 if (rc != GDK_SUCCEED) {
    1527           0 :                                         bat_destroy(orig);
    1528           0 :                                         bat_destroy(b);
    1529           0 :                                         goto bailout;
    1530             :                                 }
    1531         104 :                                 if ((rc = BUNappend(lg->del, &orig->batCacheid, false)) != GDK_SUCCEED ||
    1532          52 :                                         (rc = BUNappend(lg->add, &b->batCacheid, false)) != GDK_SUCCEED) {
    1533           0 :                                         bat_destroy(orig);
    1534           0 :                                         bat_destroy(b);
    1535           0 :                                         goto bailout;
    1536             :                                 }
    1537             :                                 rc = GDK_FAIL;
    1538          52 :                                 BBPretain(orig->batCacheid);
    1539          52 :                                 BBPretain(b->batCacheid);
    1540          52 :                                 switch (tables[i].newid) {
    1541           8 :                                 case 2002:              /* sys.schemas.id */
    1542           8 :                                         bat_destroy(bats[0].idbat);
    1543           8 :                                         bats[0].idbat = b;
    1544           8 :                                         BBPfix(b->batCacheid);
    1545           8 :                                         break;
    1546           8 :                                 case 2068:              /* sys._tables.id */
    1547           8 :                                         bat_destroy(bats[1].idbat);
    1548           8 :                                         bats[1].idbat = b;
    1549           8 :                                         BBPfix(b->batCacheid);
    1550           8 :                                         break;
    1551           8 :                                 case 2070:              /* sys._tables.schema_id */
    1552           8 :                                         bat_destroy(bats[1].parbat);
    1553           8 :                                         bats[1].parbat = b;
    1554           8 :                                         BBPfix(b->batCacheid);
    1555           8 :                                         break;
    1556           8 :                                 case 2077:              /* sys._columns.id */
    1557           8 :                                         bat_destroy(bats[2].idbat);
    1558           8 :                                         bats[2].idbat = b;
    1559           8 :                                         BBPfix(b->batCacheid);
    1560           8 :                                         break;
    1561           8 :                                 case 2082:              /* sys._columns.table_id */
    1562           8 :                                         bat_destroy(bats[2].parbat);
    1563           8 :                                         bats[2].parbat = b;
    1564           8 :                                         BBPfix(b->batCacheid);
    1565           8 :                                         break;
    1566             :                                 }
    1567          52 :                                 bat_destroy(orig);
    1568             :                         }
    1569             :                         /* now b contains the updated values for the column in tables[i] */
    1570             :                 }
    1571             :                 /* here, b is either the original, unchanged bat or the updated one */
    1572        1584 :                 if ((rc = BUNappend(lg->lg->catalog_bid, &b->batCacheid, false)) != GDK_SUCCEED ||
    1573         792 :                         (rc = BUNappend(lg->lg->catalog_id, &tables[i].newid, false)) != GDK_SUCCEED) {
    1574           0 :                         bat_destroy(b);
    1575           0 :                         goto bailout;
    1576             :                 }
    1577             :                 rc = GDK_FAIL;
    1578         792 :                 BBPretain(b->batCacheid);
    1579         792 :                 bat_destroy(b);
    1580             :         }
    1581             : 
    1582             :         /* add all extant non-system bats to the new catalog */
    1583             :         BAT *cands, *b;
    1584           8 :         if (BATcount(lg->dcatalog) == 0) {
    1585             :                 cands = NULL;
    1586             :         } else {
    1587           4 :                 if ((rc = BATsort(&b, NULL, NULL, lg->dcatalog, NULL, NULL, false, false, false)) != GDK_SUCCEED)
    1588           0 :                         goto bailout;
    1589             :                 rc = GDK_FAIL;
    1590           4 :                 cands = BATnegcands(BATcount(lg->catalog_oid), b);
    1591           4 :                 bat_destroy(b);
    1592           4 :                 if (cands == NULL)
    1593           0 :                         goto bailout;
    1594             :         }
    1595           8 :         b = BATselect(lg->catalog_oid, cands, &(lng){0}, NULL, true, true, true);
    1596           8 :         bat_destroy(cands);
    1597           8 :         if (b == NULL)
    1598           0 :                 goto bailout;
    1599             :         cands = b;
    1600           8 :         b = BATconvert(lg->catalog_oid, cands, TYPE_int, true, 0, 0, 0);
    1601           8 :         if (b == NULL) {
    1602           0 :                 bat_destroy(cands);
    1603           0 :                 goto bailout;
    1604             :         }
    1605          16 :         if ((rc = BATappend(lg->lg->catalog_id, b, NULL, false)) != GDK_SUCCEED ||
    1606           8 :                 (rc = BATappend(lg->lg->catalog_bid, lg->catalog_bid, cands, false)) != GDK_SUCCEED) {
    1607           0 :                 bat_destroy(cands);
    1608           0 :                 bat_destroy(b);
    1609           0 :                 goto bailout;
    1610             :         }
    1611             :         rc = GDK_FAIL;
    1612             :         const int *bids;
    1613           8 :         bids = (const int *) Tloc(lg->lg->catalog_bid, lg->lg->catalog_bid->batCount - BATcount(cands));
    1614        1370 :         for (BUN j = BATcount(cands), i = 0; i < j; i++)
    1615        1362 :                 BBPretain(bids[i]);
    1616           8 :         bat_destroy(cands);
    1617           8 :         bat_destroy(b);
    1618             : 
    1619             :         /* convert deleted rows bats (catalog id equals table id) from list
    1620             :          * of deleted rows to mask of deleted rows */
    1621             :         BAT *tabs;
    1622             :         /* 2164 is the largest fixed id, so select anything larger */
    1623           8 :         tabs = BATselect(lg->lg->catalog_id, NULL, &(int){2164}, &int_nil, false, true, false);
    1624           8 :         if (tabs == NULL)
    1625           0 :                 goto bailout;
    1626             :         BAT *b1;
    1627             :         /* extract those rows that refer to a known table (in bats[1].idbat) */
    1628           8 :         b1 = BATintersect(lg->lg->catalog_id, bats[1].idbat, tabs, bats[1].cands, false, false, BUN_NONE);
    1629           8 :         bat_destroy(tabs);
    1630           8 :         if (b1 == NULL)
    1631           0 :                 goto bailout;
    1632             :         BAT *b3, *b4;
    1633             :         /* find a column (any column) in each of the tables */
    1634           8 :         if ((rc = BATsemijoin(&b3, &b4, lg->lg->catalog_id, bats[2].parbat, b1, bats[2].cands, false, false, BUN_NONE)) != GDK_SUCCEED) {
    1635           0 :                 bat_destroy(b1);
    1636           0 :                 goto bailout;
    1637             :         }
    1638             :         rc = GDK_FAIL;
    1639           8 :         bat_destroy(b3);
    1640             :         /* extract column id */
    1641           8 :         b3 = BATproject(b4, bats[2].idbat);
    1642           8 :         bat_destroy(b4);
    1643           8 :         if (b3 == NULL) {
    1644           0 :                 bat_destroy(b1);
    1645           0 :                 goto bailout;
    1646             :         }
    1647             :         BAT *b2;
    1648           8 :         rc = BATleftjoin(&b2, &b4, b3, lg->lg->catalog_id, NULL, NULL, false, BUN_NONE);
    1649           8 :         bat_destroy(b3);
    1650           8 :         if (rc != GDK_SUCCEED) {
    1651           0 :                 bat_destroy(b1);
    1652           0 :                 goto bailout;
    1653             :         }
    1654           8 :         bat_destroy(b2);
    1655             :         struct canditer ci;
    1656           8 :         canditer_init(&ci, lg->lg->catalog_bid, b1);
    1657             :         const oid *cbids;
    1658           8 :         bids = Tloc(lg->lg->catalog_bid, 0);
    1659           8 :         cbids = Tloc(b4, 0);
    1660         234 :         for (BUN i = 0; i < ci.ncand; i++) {
    1661         226 :                 bat cbid = bids[cbids[i]];
    1662         226 :                 b = temp_descriptor(cbid);
    1663         226 :                 if (b == NULL) {
    1664           0 :                         bat_destroy(b1);
    1665           0 :                         bat_destroy(b3);
    1666           0 :                         goto bailout;
    1667             :                 }
    1668             :                 BUN len;
    1669         226 :                 len = BATcount(b);
    1670         226 :                 bat_destroy(b);
    1671             :                 oid o;
    1672         226 :                 o = canditer_next(&ci);
    1673             :                 bat tbid;
    1674         226 :                 tbid = bids[o - lg->lg->catalog_bid->hseqbase];
    1675         226 :                 b = temp_descriptor(tbid);
    1676             :                 BAT *bn;
    1677         226 :                 bn = BATconstant(0, TYPE_msk, &(msk){false}, len, PERSISTENT);
    1678         226 :                 if (b == NULL || bn == NULL) {
    1679           0 :                         bat_destroy(b);
    1680           0 :                         bat_destroy(bn);
    1681           0 :                         bat_destroy(b1);
    1682           0 :                         bat_destroy(b3);
    1683           0 :                         goto bailout;
    1684             :                 }
    1685             :                 const oid *dels;
    1686         226 :                 dels = Tloc(b, 0);
    1687         587 :                 for (BUN q = BUNlast(b), p = 0; p < q; p++) {
    1688         361 :                         mskSetVal(bn, (BUN) dels[p], true);
    1689             :                 }
    1690         226 :                 bat_destroy(b);
    1691         452 :                 if ((rc = BUNappend(lg->del, &tbid, false)) != GDK_SUCCEED ||
    1692         452 :                     (rc = BUNappend(lg->add, &bn->batCacheid, false)) != GDK_SUCCEED ||
    1693         226 :                     (rc = BUNreplace(lg->lg->catalog_bid, o, &bn->batCacheid, false)) != GDK_SUCCEED) {
    1694           0 :                         bat_destroy(bn);
    1695           0 :                         bat_destroy(b1);
    1696           0 :                         bat_destroy(b3);
    1697           0 :                         goto bailout;
    1698             :                 }
    1699             :                 rc = GDK_FAIL;
    1700             :                 /* moving tbid from lg->lg->catalog_bid to lg->del does not change
    1701             :                  * lrefs of tbid (old location is overwritten by new table id) */
    1702         226 :                 BBPretain(bn->batCacheid);
    1703         226 :                 BBPretain(bn->batCacheid); /* yep, twice */
    1704         226 :                 bat_destroy(bn);
    1705             :         }
    1706           8 :         bat_destroy(b1);
    1707           8 :         bat_destroy(b4);
    1708             : 
    1709             :         /* map schema/table/column ids in other system tables */
    1710           8 :         if (mapold) {
    1711             :                 /* select tables in sys schema */
    1712           8 :                 b1 = BATselect(bats[1].parbat, bats[1].cands, &(int){2000}, NULL, true, true, false);
    1713           8 :                 if (b1 == NULL)
    1714           0 :                         goto bailout;
    1715           8 :                 bids = Tloc(lg->lg->catalog_bid, 0);
    1716          40 :                 for (int i = 0; mapids[i].column != NULL; i++) {
    1717             :                         /* row ids for table in sys schema */
    1718          32 :                         BAT *b2 = BATselect(bats[1].nmbat, b1, mapids[i].table, NULL, true, true, false);
    1719          32 :                         if (b2 == NULL) {
    1720           0 :                                 bat_destroy(b1);
    1721           0 :                                 goto bailout;
    1722             :                         }
    1723             :                         /* table ids for table */
    1724          32 :                         b3 = BATproject(b2, bats[1].idbat);
    1725          32 :                         bat_destroy(b2);
    1726          32 :                         if (b3 == NULL) {
    1727           0 :                                 bat_destroy(b1);
    1728           0 :                                 goto bailout;
    1729             :                         }
    1730             :                         /* row ids for columns of table */
    1731          32 :                         b2 = BATintersect(bats[2].parbat, b3, NULL, NULL, false, false, BUN_NONE);
    1732          32 :                         bat_destroy(b3);
    1733          32 :                         if (b2 == NULL) {
    1734           0 :                                 bat_destroy(b1);
    1735           0 :                                 goto bailout;
    1736             :                         }
    1737             :                         /* row id for the column in the table we're looking for */
    1738          32 :                         b3 = BATselect(bats[2].nmbat, b2, mapids[i].column, NULL, true, true, false);
    1739          32 :                         bat_destroy(b2);
    1740          32 :                         if (b3 == NULL) {
    1741           0 :                                 bat_destroy(b1);
    1742           0 :                                 goto bailout;
    1743             :                         }
    1744             :                         /* row ids in catalog for column in table */
    1745          32 :                         b2 = BATintersect(lg->lg->catalog_id, bats[2].idbat, NULL, b3, false, false, 1);
    1746          32 :                         bat_destroy(b3);
    1747          32 :                         if (b2 == NULL) {
    1748           0 :                                 bat_destroy(b1);
    1749           0 :                                 goto bailout;
    1750             :                         }
    1751          64 :                         for (BUN j = 0; j < BATcount(b2); j++) {
    1752          32 :                                 oid p = BUNtoid(b2, j);
    1753          32 :                                 b3 = BATdescriptor(bids[p]);
    1754          32 :                                 if (b3 == NULL) {
    1755           0 :                                         bat_destroy(b1);
    1756           0 :                                         bat_destroy(b2);
    1757           0 :                                         goto bailout;
    1758             :                                 }
    1759             :                                 BAT *b4, *b5;
    1760          32 :                                 if ((rc = BATjoin(&b4, &b5, b3, mapold, NULL, NULL, false, BUN_NONE)) != GDK_SUCCEED) {
    1761           0 :                                         bat_destroy(b1);
    1762           0 :                                         bat_destroy(b2);
    1763           0 :                                         bat_destroy(b3);
    1764           0 :                                         goto bailout;
    1765             :                                 }
    1766             :                                 rc = GDK_FAIL;
    1767          32 :                                 if (BATcount(b4) == 0) {
    1768          24 :                                         bat_destroy(b3);
    1769          24 :                                         bat_destroy(b4);
    1770          24 :                                         bat_destroy(b5);
    1771             :                                 } else {
    1772             :                                         BAT *b6;
    1773           8 :                                         b6 = COLcopy(b3, b3->ttype, true, PERSISTENT);
    1774           8 :                                         bat_destroy(b3);
    1775           8 :                                         b3 = BATproject(b5, mapnew);
    1776           8 :                                         bat_destroy(b5);
    1777           8 :                                         if (b3 == NULL || b6 == NULL) {
    1778           0 :                                                 bat_destroy(b1);
    1779           0 :                                                 bat_destroy(b2);
    1780           0 :                                                 bat_destroy(b3);
    1781           0 :                                                 bat_destroy(b4);
    1782           0 :                                                 bat_destroy(b6);
    1783           0 :                                                 goto bailout;
    1784             :                                         }
    1785          16 :                                         if ((rc = BATreplace(b6, b4, b3, false)) == GDK_SUCCEED &&
    1786          16 :                                                 (rc = BUNappend(lg->del, &bids[p], false)) == GDK_SUCCEED &&
    1787           8 :                                                 (rc = BUNappend(lg->add, &b6->batCacheid, false)) == GDK_SUCCEED)
    1788           8 :                                                 rc = BUNreplace(lg->lg->catalog_bid, p, &b6->batCacheid, false);
    1789           8 :                                         BBPretain(b6->batCacheid);
    1790           8 :                                         BBPretain(b6->batCacheid);
    1791           8 :                                         bat_destroy(b3);
    1792           8 :                                         bat_destroy(b4);
    1793           8 :                                         bat_destroy(b6);
    1794           8 :                                         if (rc != GDK_SUCCEED) {
    1795           0 :                                                 bat_destroy(b1);
    1796           0 :                                                 bat_destroy(b2);
    1797           0 :                                                 goto bailout;
    1798             :                                         }
    1799             :                                         rc = GDK_FAIL;
    1800             :                                 }
    1801             :                         }
    1802          32 :                         bat_destroy(b2);
    1803             :                 }
    1804           8 :                 bat_destroy(b1);
    1805             :         }
    1806             : 
    1807             :         rc = GDK_SUCCEED;
    1808             : 
    1809           8 :   bailout:
    1810           8 :         bat_destroy(mapold);
    1811           8 :         bat_destroy(mapnew);
    1812          32 :         for (int i = 0; i < 3; i++) {
    1813          24 :                 bat_destroy(bats[i].nmbat);
    1814          24 :                 bat_destroy(bats[i].idbat);
    1815          24 :                 bat_destroy(bats[i].parbat);
    1816          24 :                 bat_destroy(bats[i].cands);
    1817             :         }
    1818           8 :         return rc;
    1819             : }
    1820             : 
    1821             : static gdk_return
    1822           8 : bl_postversion(void *Store, old_logger *old_lg)
    1823             : {
    1824             :         sqlstore *store = Store;
    1825             :         (void)store;
    1826           8 :         if (store->catalog_version < 52300 && upgrade(old_lg) != GDK_SUCCEED)
    1827             :                 return GDK_FAIL;
    1828           8 :         logger *lg = old_lg->lg;
    1829             :         bool tabins_first = true;
    1830             : 
    1831             : #ifdef CATALOG_NOV2019
    1832           8 :         if (store->catalog_version <= CATALOG_NOV2019) {
    1833             :                 BAT *te, *tne;
    1834             :                 const int *ocl; /* old eclass */
    1835             :                 int *ncl;       /* new eclass */
    1836             : 
    1837           0 :                 te = temp_descriptor(logger_find_bat(lg, 2014)); /* sys.types.eclass */
    1838           0 :                 if (te == NULL)
    1839             :                         return GDK_FAIL;
    1840           0 :                 tne = COLnew(te->hseqbase, TYPE_int, BATcount(te), PERSISTENT);
    1841           0 :                 if (tne == NULL) {
    1842           0 :                         bat_destroy(te);
    1843           0 :                         return GDK_FAIL;
    1844             :                 }
    1845           0 :                 ocl = Tloc(te, 0);
    1846           0 :                 ncl = Tloc(tne, 0);
    1847           0 :                 for (BUN p = 0, q = BUNlast(te); p < q; p++) {
    1848           0 :                         switch (ocl[p]) {
    1849           0 :                         case EC_TIME_TZ:                /* old EC_DATE */
    1850           0 :                                 ncl[p] = EC_DATE;
    1851           0 :                                 break;
    1852           0 :                         case EC_DATE:                   /* old EC_TIMESTAMP */
    1853           0 :                                 ncl[p] = EC_TIMESTAMP;
    1854           0 :                                 break;
    1855           0 :                         case EC_TIMESTAMP:              /* old EC_GEOM */
    1856           0 :                                 ncl[p] = EC_GEOM;
    1857           0 :                                 break;
    1858           0 :                         case EC_TIMESTAMP_TZ:   /* old EC_EXTERNAL */
    1859           0 :                                 ncl[p] = EC_EXTERNAL;
    1860           0 :                                 break;
    1861           0 :                         default:
    1862             :                                 /* others stay unchanged */
    1863           0 :                                 ncl[p] = ocl[p];
    1864           0 :                                 break;
    1865             :                         }
    1866             :                 }
    1867           0 :                 BATsetcount(tne, BATcount(te));
    1868           0 :                 tne->tnil = false;
    1869           0 :                 tne->tnonil = true;
    1870           0 :                 tne->tsorted = false;
    1871           0 :                 tne->trevsorted = false;
    1872           0 :                 tne->tkey = false;
    1873           0 :                 if (BUNappend(old_lg->del, &te->batCacheid, false) != GDK_SUCCEED ||
    1874           0 :                         BUNappend(old_lg->add, &tne->batCacheid, false) != GDK_SUCCEED ||
    1875           0 :                         BUNreplace(lg->catalog_bid, BUNfnd(lg->catalog_id, &(int){2014}), &tne->batCacheid, false) != GDK_SUCCEED) {
    1876           0 :                         bat_destroy(te);
    1877           0 :                         bat_destroy(tne);
    1878           0 :                         return GDK_FAIL;
    1879             :                 }
    1880           0 :                 BBPretain(tne->batCacheid);
    1881           0 :                 BBPretain(tne->batCacheid);
    1882           0 :                 bat_destroy(te);
    1883           0 :                 bat_destroy(tne);
    1884             :         }
    1885             : #endif
    1886             : 
    1887             : #ifdef CATALOG_JUN2020
    1888           8 :         if (store->catalog_version <= CATALOG_JUN2020
    1889             : #ifdef CATALOG_JUN2020_MMT
    1890           8 :                 || store->catalog_version == CATALOG_JUN2020_MMT
    1891             : #endif
    1892             :                 ) {
    1893             :                 BAT *b;                                                          /* temp variable */
    1894             :                 {
    1895             :                         /* new BOOLEAN column sys.functions.semantics */
    1896           0 :                         b = temp_descriptor(logger_find_bat(lg, 2017)); /* sys.functions.id */
    1897           0 :                         BAT *sem = BATconstant(b->hseqbase, TYPE_bit, &(bit){1}, BATcount(b), PERSISTENT);
    1898           0 :                         bat_destroy(b);
    1899           0 :                         if (sem == NULL)
    1900           0 :                                 return GDK_FAIL;
    1901           0 :                         if ((sem = BATsetaccess(sem, BAT_READ)) == NULL ||
    1902             :                                 /* 2162 is sys.functions.semantics */
    1903           0 :                                 BUNappend(lg->catalog_id, &(int) {2162}, false) != GDK_SUCCEED ||
    1904           0 :                                 BUNappend(lg->catalog_bid, &sem->batCacheid, false) != GDK_SUCCEED ||
    1905           0 :                                 BUNappend(old_lg->add, &sem->batCacheid, false) != GDK_SUCCEED) {
    1906           0 :                                 bat_destroy(sem);
    1907           0 :                                 return GDK_FAIL;
    1908             :                         }
    1909           0 :                         BBPretain(sem->batCacheid);
    1910           0 :                         BBPretain(sem->batCacheid); /* yep, twice */
    1911           0 :                         bat_destroy(sem);
    1912           0 :                         if (tabins(lg, old_lg, tabins_first, -1, 0,
    1913           0 :                                            2076, &(msk) {false},    /* sys._columns */
    1914             :                                            /* 2162 is sys.functions.semantics */
    1915           0 :                                            2077, &(int) {2162},             /* sys._columns.id */
    1916             :                                            2078, "semantics",         /* sys._columns.name */
    1917             :                                            2079, "boolean",                   /* sys._columns.type */
    1918           0 :                                            2080, &(int) {1},                /* sys._columns.type_digits */
    1919           0 :                                            2081, &(int) {0},                /* sys._columns.type_scale */
    1920             :                                            /* 2016 is sys.functions */
    1921           0 :                                            2082, &(int) {2016},             /* sys._columns.table_id */
    1922             :                                            2083, str_nil,                       /* sys._columns.default */
    1923           0 :                                            2084, &(bit) {TRUE},             /* sys._columns.null */
    1924           0 :                                            2085, &(int) {11},               /* sys._columns.number */
    1925             :                                            2086, str_nil,                       /* sys._columns.storage */
    1926             :                                            0) != GDK_SUCCEED)
    1927           0 :                                 return GDK_FAIL;
    1928             :                         tabins_first = false;
    1929             :                 }
    1930             : 
    1931             :                 /* sys.functions i.e. deleted rows */
    1932           0 :                 BAT *del_funcs = temp_descriptor(logger_find_bat(lg, 2016));
    1933             :                 {
    1934             :                         /* move sql.degrees, sql.radians, sql.like and sql.ilike functions
    1935             :                          * from 09_like.sql and 10_math.sql script to sql_types list */
    1936             :                         /* sys.functions.name */
    1937           0 :                         BAT *func_func = temp_descriptor(logger_find_bat(lg, 2018));
    1938             :                         /* sys.functions.schema_id */
    1939           0 :                         BAT *func_schem = temp_descriptor(logger_find_bat(lg, 2026));
    1940             :                         BAT *func_tid;
    1941             :                         BAT *cands;
    1942           0 :                         if (del_funcs == NULL || func_func == NULL || func_schem == NULL) {
    1943           0 :                                 bat_destroy(del_funcs);
    1944           0 :                                 bat_destroy(func_func);
    1945           0 :                                 bat_destroy(func_schem);
    1946           0 :                                 return GDK_FAIL;
    1947             :                         }
    1948           0 :                         func_tid = BATmaskedcands(0, BATcount(del_funcs), del_funcs, false);
    1949           0 :                         if (func_tid == NULL) {
    1950           0 :                                 bat_destroy(del_funcs);
    1951           0 :                                 bat_destroy(func_func);
    1952           0 :                                 bat_destroy(func_schem);
    1953           0 :                                 return GDK_FAIL;
    1954             :                         }
    1955             :                         /* select * from sys.functions where schema_id = 2000; */
    1956           0 :                         b = BATselect(func_schem, func_tid, &(int) {2000}, NULL, true, true, false);
    1957           0 :                         bat_destroy(func_schem);
    1958           0 :                         bat_destroy(func_tid);
    1959             :                         cands = b;
    1960           0 :                         if (cands == NULL) {
    1961           0 :                                 bat_destroy(del_funcs);
    1962           0 :                                 bat_destroy(func_func);
    1963           0 :                                 return GDK_FAIL;
    1964             :                         }
    1965             : 
    1966             :                         BAT *funcs;
    1967           0 :                         if ((funcs = COLnew(0, TYPE_str, 4, TRANSIENT)) == NULL ||
    1968           0 :                                 BUNappend(funcs, "degrees", false) != GDK_SUCCEED ||
    1969           0 :                                 BUNappend(funcs, "ilike", false) != GDK_SUCCEED ||
    1970           0 :                                 BUNappend(funcs, "like", false) != GDK_SUCCEED ||
    1971           0 :                                 BUNappend(funcs, "radians", false) != GDK_SUCCEED) {
    1972           0 :                                 bat_destroy(funcs);
    1973           0 :                                 bat_destroy(del_funcs);
    1974           0 :                                 bat_destroy(func_func);
    1975           0 :                                 return GDK_FAIL;
    1976             :                         }
    1977           0 :                         b = BATintersect(func_func, funcs, cands, NULL, false, false, 4);
    1978           0 :                         bat_destroy(func_func);
    1979           0 :                         bat_destroy(funcs);
    1980           0 :                         bat_destroy(cands);
    1981             :                         funcs = NULL;
    1982             :                         gdk_return rc;
    1983             :                         rc = GDK_FAIL;
    1984           0 :                         if (b != NULL &&
    1985           0 :                                 (funcs = BATconstant(0, TYPE_msk, &(msk){true}, BATcount(b), TRANSIENT)) != NULL)
    1986           0 :                                 rc = BATreplace(del_funcs, b, funcs, false);
    1987           0 :                         bat_destroy(b);
    1988           0 :                         bat_destroy(funcs);
    1989           0 :                         if (rc != GDK_SUCCEED)
    1990             :                                 return rc;
    1991             :                 }
    1992             : 
    1993             :                 {
    1994             :                         /* Fix SQL aggregation functions defined on the wrong modules:
    1995             :                          * sql.null, sql.all, sql.zero_or_one and sql.not_unique */
    1996           0 :                         BAT *func_tid = BATmaskedcands(0, BATcount(del_funcs), del_funcs, false);
    1997             :                         /* sys.functions.mod */
    1998           0 :                         BAT *func_mod = temp_descriptor(logger_find_bat(lg, 2020));
    1999           0 :                         bat_destroy(del_funcs);
    2000           0 :                         if (func_tid == NULL || func_mod == NULL) {
    2001           0 :                                 bat_destroy(func_tid);
    2002           0 :                                 bat_destroy(func_mod);
    2003           0 :                                 return GDK_FAIL;
    2004             :                         }
    2005             : 
    2006             :                         /* find the (undeleted) functions defined on "sql" module */
    2007           0 :                         BAT *sqlfunc = BATselect(func_mod, func_tid, "sql", NULL, true, true, false);
    2008           0 :                         bat_destroy(func_tid);
    2009           0 :                         if (sqlfunc == NULL) {
    2010           0 :                                 bat_destroy(func_mod);
    2011           0 :                                 return GDK_FAIL;
    2012             :                         }
    2013             :                         /* sys.functions.type */
    2014           0 :                         BAT *func_type = temp_descriptor(logger_find_bat(lg, 2022));
    2015           0 :                         if (func_type == NULL) {
    2016           0 :                                 bat_destroy(func_mod);
    2017           0 :                                 bat_destroy(sqlfunc);
    2018           0 :                                 return GDK_FAIL;
    2019             :                         }
    2020             :                         /* and are aggregates (3) */
    2021           0 :                         BAT *sqlaggr_func = BATselect(func_type, sqlfunc, &(int) {3}, NULL, true, true, false);
    2022           0 :                         bat_destroy(sqlfunc);
    2023           0 :                         bat_destroy(func_type);
    2024           0 :                         if (sqlaggr_func == NULL) {
    2025           0 :                                 bat_destroy(func_mod);
    2026           0 :                                 return GDK_FAIL;
    2027             :                         }
    2028             : 
    2029             :                         /* sys.functions.func */
    2030           0 :                         BAT *func_func = temp_descriptor(logger_find_bat(lg, 2019));
    2031           0 :                         if (func_func == NULL) {
    2032           0 :                                 bat_destroy(func_mod);
    2033           0 :                                 bat_destroy(sqlaggr_func);
    2034           0 :                                 return GDK_FAIL;
    2035             :                         }
    2036           0 :                         b = COLcopy(func_mod, func_mod->ttype, true, PERSISTENT);
    2037           0 :                         if (b == NULL) {
    2038           0 :                                 bat_destroy(func_func);
    2039           0 :                                 bat_destroy(func_mod);
    2040           0 :                                 bat_destroy(sqlaggr_func);
    2041           0 :                                 return GDK_FAIL;
    2042             :                         }
    2043           0 :                         if (BUNappend(old_lg->del, &func_mod->batCacheid, false) != GDK_SUCCEED ||
    2044           0 :                                 BUNappend(old_lg->add, &b->batCacheid, false) != GDK_SUCCEED ||
    2045             :                                 /* 2020 is sys.functions.mod */
    2046           0 :                                 BUNreplace(lg->catalog_bid, BUNfnd(lg->catalog_id, &(int){2020}), &b->batCacheid, false) != GDK_SUCCEED) {
    2047           0 :                                 bat_destroy(func_func);
    2048           0 :                                 bat_destroy(func_mod);
    2049           0 :                                 bat_destroy(sqlaggr_func);
    2050           0 :                                 bat_destroy(b);
    2051           0 :                                 return GDK_FAIL;
    2052             :                         }
    2053           0 :                         BBPretain(b->batCacheid);
    2054           0 :                         BBPretain(b->batCacheid);
    2055           0 :                         bat_destroy(func_mod);
    2056             :                         func_mod = b;
    2057             :                         BAT *aggrs;
    2058           0 :                         if ((aggrs = COLnew(0, TYPE_str, 4, TRANSIENT)) == NULL ||
    2059           0 :                                 BUNappend(aggrs, "all", false) != GDK_SUCCEED ||
    2060           0 :                                 BUNappend(aggrs, "no_unique", false) != GDK_SUCCEED ||
    2061           0 :                                 BUNappend(aggrs, "null", false) != GDK_SUCCEED ||
    2062           0 :                                 BUNappend(aggrs, "zero_or_one", false) != GDK_SUCCEED) {
    2063           0 :                                 bat_destroy(aggrs);
    2064           0 :                                 bat_destroy(func_func);
    2065           0 :                                 bat_destroy(func_mod);
    2066           0 :                                 bat_destroy(sqlaggr_func);
    2067           0 :                                 return GDK_FAIL;
    2068             :                         }
    2069           0 :                         b = BATintersect(func_func, aggrs, sqlaggr_func, NULL, false, false, 4);
    2070           0 :                         bat_destroy(func_func);
    2071           0 :                         bat_destroy(aggrs);
    2072           0 :                         bat_destroy(sqlaggr_func);
    2073             :                         aggrs = NULL;
    2074             :                         gdk_return rc = GDK_FAIL;
    2075           0 :                         if (b != NULL &&
    2076           0 :                                 (aggrs = BATconstant(0, TYPE_str, "aggr", BATcount(b), TRANSIENT)) != NULL)
    2077           0 :                                 rc = BATreplace(func_mod, b, aggrs, false);
    2078           0 :                         bat_destroy(b);
    2079           0 :                         bat_destroy(aggrs);
    2080           0 :                         bat_destroy(func_mod);
    2081           0 :                         if (rc != GDK_SUCCEED)
    2082             :                                 return rc;
    2083             :                 }
    2084             :         }
    2085             : #endif
    2086             : 
    2087             : #ifdef CATALOG_OCT2020
    2088           8 :         if (store->catalog_version <= CATALOG_OCT2020) { /* not for Jun2020-mmt! */
    2089             :                 /* add sub column to "objects" table. This is required for merge tables */
    2090             :                 /* alter table sys.objects add column sub integer; */
    2091           8 :                 if (tabins(lg, old_lg, tabins_first, -1, 0,
    2092           8 :                                    2076, &(msk) {false},    /* sys._columns */
    2093             :                                    /* 2163 is sys.objects.sub */
    2094           8 :                                    2077, &(int) {2163},             /* sys._columns.id */
    2095             :                                    2078, "sub",                               /* sys._columns.name */
    2096             :                                    2079, "int",                               /* sys._columns.type */
    2097           8 :                                    2080, &(int) {32},               /* sys._columns.type_digits */
    2098           8 :                                    2081, &(int) {0},                /* sys._columns.type_scale */
    2099             :                                    /* 2110 is sys.objects */
    2100           8 :                                    2082, &(int) {2110},             /* sys._columns.table_id */
    2101             :                                    2083, str_nil,                       /* sys._columns.default */
    2102           8 :                                    2084, &(bit) {TRUE},             /* sys._columns.null */
    2103           8 :                                    2085, &(int) {3},                /* sys._columns.number */
    2104             :                                    2086, str_nil,                       /* sys._columns.storage */
    2105             :                                    0) != GDK_SUCCEED)
    2106           0 :                         return GDK_FAIL;
    2107             :                 tabins_first = false;
    2108           8 :                 if (tabins(lg, old_lg, tabins_first, -1, 0,
    2109           8 :                                    2076, &(msk) {false},    /* sys._columns */
    2110             :                                    /* 2164 is tmp.objects.sub */
    2111           8 :                                    2077, &(int) {2164},             /* sys._columns.id */
    2112             :                                    2078, "sub",                               /* sys._columns.name */
    2113             :                                    2079, "int",                               /* sys._columns.type */
    2114           8 :                                    2080, &(int) {32},               /* sys._columns.type_digits */
    2115           8 :                                    2081, &(int) {0},                /* sys._columns.type_scale */
    2116             :                                    /* 2158 is tmp.objects */
    2117           8 :                                    2082, &(int) {2158},             /* sys._columns.table_id */
    2118             :                                    2083, str_nil,                       /* sys._columns.default */
    2119           8 :                                    2084, &(bit) {TRUE},             /* sys._columns.null */
    2120           8 :                                    2085, &(int) {3},                /* sys._columns.number */
    2121             :                                    2086, str_nil,                       /* sys._columns.storage */
    2122             :                                    0) != GDK_SUCCEED)
    2123           0 :                         return GDK_FAIL;
    2124             : 
    2125             :                 /* create bat for new column sys.objects.sub with value NULL, then
    2126             :                  * update sys.objects set sub = nr, nr = id where nr > 2000 */
    2127             :                 {
    2128           8 :                         BAT *objs_id = temp_descriptor(logger_find_bat(lg, 2111)); /* sys.objects.id */
    2129           8 :                         BAT *objs_nr = temp_descriptor(logger_find_bat(lg, 2113)); /* sys.objects.nr */
    2130           8 :                         BAT *objs_sub = BATconstant(objs_id->hseqbase, TYPE_int, &int_nil, BATcount(objs_id), PERSISTENT);
    2131           8 :                         BAT *b = temp_descriptor(logger_find_bat(lg, 2110)); /* sys.objects */
    2132           8 :                         if (objs_id == NULL || objs_nr == NULL || objs_sub == NULL || b == NULL) {
    2133           0 :                                 bat_destroy(objs_id);
    2134           0 :                                 bat_destroy(objs_nr);
    2135           0 :                                 bat_destroy(b);
    2136           0 :                                 bat_destroy(objs_sub);
    2137           0 :                                 return GDK_FAIL;
    2138             :                         }
    2139           8 :                         BAT *cands = BATmaskedcands(0, BATcount(b), b, false);
    2140           8 :                         bat_destroy(b);
    2141           8 :                         if (cands == NULL) {
    2142           0 :                                 bat_destroy(objs_id);
    2143           0 :                                 bat_destroy(objs_nr);
    2144           0 :                                 bat_destroy(objs_sub);
    2145           0 :                                 return GDK_FAIL;
    2146             :                         }
    2147           8 :                         b = BATselect(objs_nr, cands, &(int) {2000}, &int_nil, false, false, false);
    2148           8 :                         bat_destroy(cands);
    2149           8 :                         if (b == NULL) {
    2150           0 :                                 bat_destroy(objs_id);
    2151           0 :                                 bat_destroy(objs_nr);
    2152           0 :                                 bat_destroy(objs_sub);
    2153           0 :                                 return GDK_FAIL;
    2154             :                         }
    2155             :                         cands = b;
    2156           8 :                         b = BATproject2(cands, objs_nr, NULL);
    2157           8 :                         if (b == NULL) {
    2158           0 :                                 bat_destroy(objs_id);
    2159           0 :                                 bat_destroy(objs_nr);
    2160           0 :                                 bat_destroy(objs_sub);
    2161           0 :                                 bat_destroy(cands);
    2162           0 :                                 return GDK_FAIL;
    2163             :                         }
    2164           8 :                         gdk_return rc = BATreplace(objs_sub, cands, b, false);
    2165           8 :                         bat_destroy(b);
    2166           8 :                         if (rc != GDK_SUCCEED) {
    2167           0 :                                 bat_destroy(objs_id);
    2168           0 :                                 bat_destroy(objs_nr);
    2169           0 :                                 bat_destroy(objs_sub);
    2170           0 :                                 bat_destroy(cands);
    2171           0 :                                 return GDK_FAIL;
    2172             :                         }
    2173           8 :                         b = COLcopy(objs_nr, objs_nr->ttype, true, PERSISTENT);
    2174           8 :                         rc = BUNappend(old_lg->del, &objs_nr->batCacheid, false);
    2175           8 :                         bat_destroy(objs_nr);
    2176           8 :                         if (b == NULL || rc != GDK_SUCCEED) {
    2177           0 :                                 bat_destroy(objs_id);
    2178           0 :                                 bat_destroy(objs_sub);
    2179           0 :                                 bat_destroy(cands);
    2180           0 :                                 bat_destroy(b);
    2181           0 :                                 return GDK_FAIL;
    2182             :                         }
    2183             :                         objs_nr = b;
    2184           8 :                         if (BUNappend(old_lg->add, &objs_nr->batCacheid, false) != GDK_SUCCEED) {
    2185           0 :                                 bat_destroy(objs_id);
    2186           0 :                                 bat_destroy(objs_sub);
    2187           0 :                                 bat_destroy(objs_nr);
    2188           0 :                                 bat_destroy(cands);
    2189           0 :                                 return GDK_FAIL;
    2190             :                         }
    2191           8 :                         BBPretain(objs_nr->batCacheid);
    2192           8 :                         b = BATproject2(cands, objs_id, NULL);
    2193           8 :                         if (b == NULL) {
    2194           0 :                                 bat_destroy(objs_id);
    2195           0 :                                 bat_destroy(objs_nr);
    2196           0 :                                 bat_destroy(objs_sub);
    2197           0 :                                 bat_destroy(cands);
    2198           0 :                                 return GDK_FAIL;
    2199             :                         }
    2200           8 :                         rc = BATreplace(objs_nr, cands, b, false);
    2201           8 :                         bat_destroy(b);
    2202           8 :                         if (rc != GDK_SUCCEED) {
    2203           0 :                                 bat_destroy(objs_id);
    2204           0 :                                 bat_destroy(objs_nr);
    2205           0 :                                 bat_destroy(objs_sub);
    2206           0 :                                 bat_destroy(cands);
    2207           0 :                                 return GDK_FAIL;
    2208             :                         }
    2209             : 
    2210           8 :                         b = COLcopy(objs_id, objs_id->ttype, true, PERSISTENT);
    2211           8 :                         rc = BUNappend(old_lg->del, &objs_id->batCacheid, false);
    2212           8 :                         bat_destroy(objs_id);
    2213           8 :                         if (b == NULL || rc != GDK_SUCCEED) {
    2214           0 :                                 bat_destroy(objs_nr);
    2215           0 :                                 bat_destroy(objs_sub);
    2216           0 :                                 bat_destroy(cands);
    2217           0 :                                 bat_destroy(b);
    2218           0 :                                 return GDK_FAIL;
    2219             :                         }
    2220             :                         objs_id = b;
    2221           8 :                         if (BUNappend(old_lg->add, &objs_id->batCacheid, false) != GDK_SUCCEED) {
    2222           0 :                                 bat_destroy(objs_id);
    2223           0 :                                 bat_destroy(objs_nr);
    2224           0 :                                 bat_destroy(objs_sub);
    2225           0 :                                 bat_destroy(cands);
    2226           0 :                                 return GDK_FAIL;
    2227             :                         }
    2228           8 :                         BBPretain(objs_id->batCacheid);
    2229             : 
    2230           8 :                         BUN cnt = BATcount(cands), p;
    2231             : 
    2232          16 :                         if (!(b = COLnew(objs_id->hseqbase, TYPE_int, cnt, TRANSIENT)) ||
    2233           8 :                                 (p = BUNfnd(old_lg->seqs_id, &(int){OBJ_SID})) == BUN_NONE) {
    2234           0 :                                 bat_destroy(objs_id);
    2235           0 :                                 bat_destroy(objs_nr);
    2236           0 :                                 bat_destroy(objs_sub);
    2237           0 :                                 bat_destroy(cands);
    2238           0 :                                 bat_destroy(b);
    2239           0 :                                 return GDK_FAIL;
    2240             :                         }
    2241             : 
    2242           8 :                         int *bp = (int*)Tloc(b, 0), id = (int) *(lng *) Tloc(old_lg->seqs_val, p);
    2243          38 :                         for (BUN i = 0; i < cnt; i++)
    2244          30 :                                 bp[i] = id++;
    2245           8 :                         BATsetcount(b, cnt);
    2246           8 :                         b->tsorted = b->trevsorted = b->tkey = b->tnonil = b->tnil = false;
    2247           8 :                         b->tnosorted = b->tnorevsorted = 0;
    2248           8 :                         lng lid = (lng) id;
    2249             : 
    2250           8 :                         rc = BATreplace(objs_id, cands, b, false);
    2251           8 :                         bat_destroy(cands);
    2252           8 :                         bat_destroy(b);
    2253           8 :                         if (rc != GDK_SUCCEED) {
    2254           0 :                                 bat_destroy(objs_id);
    2255           0 :                                 bat_destroy(objs_nr);
    2256           0 :                                 bat_destroy(objs_sub);
    2257           0 :                                 return GDK_FAIL;
    2258             :                         }
    2259             : 
    2260             :                         /* 2111 is sys.objects.id and 2113 is sys.objects.nr */
    2261          16 :                         if (BUNreplace(old_lg->seqs_val, BUNfnd(old_lg->seqs_id, &(int){OBJ_SID}), &lid, false) != GDK_SUCCEED ||
    2262          16 :                                 BUNreplace(lg->catalog_bid, BUNfnd(lg->catalog_id, &(int){2111}), &objs_id->batCacheid, false) != GDK_SUCCEED ||
    2263          16 :                                 BUNreplace(lg->catalog_bid, BUNfnd(lg->catalog_id, &(int){2113}), &objs_nr->batCacheid, false) != GDK_SUCCEED ||
    2264          16 :                                 (objs_sub = BATsetaccess(objs_sub, BAT_READ)) == NULL ||
    2265          16 :                                 (objs_id = BATsetaccess(objs_id, BAT_READ)) == NULL ||
    2266          16 :                                 (objs_nr = BATsetaccess(objs_nr, BAT_READ)) == NULL ||
    2267             :                                 /* 2163 is sys.objects.sub */
    2268          16 :                                 BUNappend(lg->catalog_id, &(int) {2163}, false) != GDK_SUCCEED ||
    2269          16 :                                 BUNappend(old_lg->add, &objs_sub->batCacheid, false) != GDK_SUCCEED ||
    2270           8 :                                 BUNappend(lg->catalog_bid, &objs_sub->batCacheid, false) != GDK_SUCCEED) {
    2271           0 :                                 bat_destroy(objs_id);
    2272           0 :                                 bat_destroy(objs_nr);
    2273           0 :                                 bat_destroy(objs_sub);
    2274           0 :                                 return GDK_FAIL;
    2275             :                         }
    2276           8 :                         BBPretain(objs_sub->batCacheid);
    2277           8 :                         BBPretain(objs_sub->batCacheid);
    2278           8 :                         BBPretain(objs_nr->batCacheid);
    2279           8 :                         BBPretain(objs_id->batCacheid);
    2280           8 :                         bat_destroy(objs_id);
    2281           8 :                         bat_destroy(objs_nr);
    2282           8 :                         bat_destroy(objs_sub);
    2283             : 
    2284             :                         /* alter table tmp.objects add column sub integer; */
    2285           8 :                         objs_sub = BATconstant(0, TYPE_int, &int_nil, 0, PERSISTENT);
    2286           8 :                         if (objs_sub == NULL) {
    2287             :                                 return GDK_FAIL;
    2288             :                         }
    2289          16 :                         if ((objs_sub = BATsetaccess(objs_sub, BAT_READ)) == NULL ||
    2290             :                                 /* 2164 is tmp.objects.sub */
    2291          16 :                                 BUNappend(lg->catalog_id, &(int) {2164}, false) != GDK_SUCCEED ||
    2292          16 :                                 BUNappend(old_lg->add, &objs_sub->batCacheid, false) != GDK_SUCCEED ||
    2293           8 :                                 BUNappend(lg->catalog_bid, &objs_sub->batCacheid, false) != GDK_SUCCEED) {
    2294           0 :                                 bat_destroy(objs_sub);
    2295           0 :                                 return GDK_FAIL;
    2296             :                         }
    2297           8 :                         BBPretain(objs_sub->batCacheid);
    2298           8 :                         BBPretain(objs_sub->batCacheid);
    2299           8 :                         bat_destroy(objs_sub);
    2300             :                 }
    2301             :         }
    2302             : #endif
    2303             : #ifdef CATALOG_OCT2020
    2304           8 :         if (store->catalog_version <= CATALOG_OCT2020
    2305             : #ifdef CATALOG_JUN2020_MMT
    2306             :                 || store->catalog_version == CATALOG_JUN2020_MMT
    2307             : #endif
    2308             :                 ) {
    2309             :                 /* update sys.functions set mod = 'sql' where mod = 'user'; */
    2310             :                 {
    2311             :                         BAT *b1, *b2, *b3;
    2312           8 :                         b1 = temp_descriptor(logger_find_bat(lg, 2016)); /* sys.functions */
    2313           8 :                         if (b1 == NULL)
    2314             :                                 return GDK_FAIL;
    2315             :                         /* undeleted rows of sys.functions */
    2316           8 :                         b2 = BATmaskedcands(0, BATcount(b1), b1, false);
    2317           8 :                         b3 = temp_descriptor(logger_find_bat(lg, 2020)); /* sys.functions.mod */
    2318           8 :                         bat_destroy(b1);
    2319           8 :                         if (b2 == NULL || b3 == NULL) {
    2320           0 :                                 bat_destroy(b2);
    2321           0 :                                 bat_destroy(b3);
    2322           0 :                                 return GDK_FAIL;
    2323             :                         }
    2324             :                         /* mod = 'user' */
    2325           8 :                         b1 = BATselect(b3, b2, "user", NULL, true, true, false);
    2326           8 :                         bat_destroy(b2);
    2327           8 :                         if (b1 == NULL) {
    2328           0 :                                 bat_destroy(b3);
    2329           0 :                                 return GDK_FAIL;
    2330             :                         }
    2331           8 :                         if (BATcount(b1) > 0) {
    2332           8 :                                 if (BUNfnd(old_lg->add, &b3->batCacheid) == BUN_NONE) {
    2333             :                                         /* replace sys.functions.mod with a copy that we can modify */
    2334           8 :                                         b2 = COLcopy(b3, b3->ttype, true, PERSISTENT);
    2335           8 :                                         if (b2 == NULL) {
    2336           0 :                                                 bat_destroy(b1);
    2337           0 :                                                 bat_destroy(b3);
    2338           0 :                                                 return GDK_FAIL;
    2339             :                                         }
    2340          16 :                                         if (BUNappend(old_lg->del, &b3->batCacheid, false) != GDK_SUCCEED ||
    2341          16 :                                                 BUNappend(old_lg->add, &b2->batCacheid, false) != GDK_SUCCEED ||
    2342             :                                                 /* 2020 is sys.functions.mod */
    2343           8 :                                                 BUNreplace(lg->catalog_bid, BUNfnd(lg->catalog_id, &(int){2020}), &b2->batCacheid, false) != GDK_SUCCEED) {
    2344           0 :                                                 bat_destroy(b1);
    2345           0 :                                                 bat_destroy(b3);
    2346           0 :                                                 bat_destroy(b2);
    2347           0 :                                                 return GDK_FAIL;
    2348             :                                         }
    2349           8 :                                         BBPretain(b2->batCacheid);
    2350           8 :                                         BBPretain(b2->batCacheid);
    2351           8 :                                         bat_destroy(b3);
    2352             :                                         b3 = b2;
    2353             :                                 }
    2354           8 :                                 b2 = BATconstant(0, TYPE_str, "sql", BATcount(b1), TRANSIENT);
    2355           8 :                                 if (b2 == NULL) {
    2356           0 :                                         bat_destroy(b1);
    2357           0 :                                         bat_destroy(b3);
    2358           0 :                                         return GDK_FAIL;
    2359             :                                 }
    2360           8 :                                 gdk_return rc = BATreplace(b3, b1, b2, false);
    2361           8 :                                 bat_destroy(b2);
    2362           8 :                                 bat_destroy(b3);
    2363           8 :                                 if (rc != GDK_SUCCEED) {
    2364           0 :                                         bat_destroy(b1);
    2365           0 :                                         return rc;
    2366             :                                 }
    2367             :                         }
    2368           8 :                         bat_destroy(b1);
    2369             :                 }
    2370             : 
    2371             :                 /* update sys.args set type = 'clob' where type = 'char' and type_digits = 0 and func_id > 2000 */
    2372             :                 {
    2373             :                         BAT *b1, *b2, *b3;
    2374           8 :                         b1 = temp_descriptor(logger_find_bat(lg, 2028)); /* sys.args */
    2375           8 :                         if (b1 == NULL)
    2376           0 :                                 return GDK_FAIL;
    2377           8 :                         b2 = BATmaskedcands(0, BATcount(b1), b1, false);
    2378           8 :                         bat_destroy(b1);
    2379           8 :                         b3 = temp_descriptor(logger_find_bat(lg, 2030)); /* sys.args.func_id */
    2380           8 :                         if (b2 == NULL || b3 == NULL) {
    2381           0 :                                 bat_destroy(b2);
    2382           0 :                                 bat_destroy(b3);
    2383           0 :                                 return GDK_FAIL;
    2384             :                         }
    2385             :                         /* func_id > 2000 */
    2386           8 :                         b1 = BATselect(b3, b2, &(int){2000}, &int_nil, false, false, false);
    2387           8 :                         bat_destroy(b2);
    2388           8 :                         bat_destroy(b3);
    2389           8 :                         b3 = temp_descriptor(logger_find_bat(lg, 2033)); /* sys.args.type_digits */
    2390           8 :                         if (b1 == NULL || b3 == NULL) {
    2391           0 :                                 bat_destroy(b1);
    2392           0 :                                 bat_destroy(b3);
    2393           0 :                                 return GDK_FAIL;
    2394             :                         }
    2395             :                         /* and type_digits = 0 */
    2396           8 :                         b2 = BATselect(b3, b1, &(int){0}, NULL, true, false, false);
    2397           8 :                         bat_destroy(b3);
    2398           8 :                         bat_destroy(b1);
    2399           8 :                         b1 = temp_descriptor(logger_find_bat(lg, 2032)); /* sys.args.type */
    2400           8 :                         if (b1 == NULL || b2 == NULL) {
    2401           0 :                                 bat_destroy(b1);
    2402           0 :                                 bat_destroy(b2);
    2403           0 :                                 return GDK_FAIL;
    2404             :                         }
    2405             :                         /* and type = 'char' */
    2406           8 :                         b3 = BATselect(b1, b2, "char", NULL, true, false, false);
    2407           8 :                         bat_destroy(b2);
    2408           8 :                         if (BATcount(b3) > 0) {
    2409           8 :                                 if (BUNfnd(old_lg->add, &b1->batCacheid) == BUN_NONE) {
    2410             :                                         /* replace sys.args.type with a copy that we can modify */
    2411           8 :                                         b2 = COLcopy(b1, b1->ttype, true, PERSISTENT);
    2412          16 :                                         if (b2 == NULL ||
    2413          16 :                                                 BUNappend(old_lg->del, &b1->batCacheid, false) != GDK_SUCCEED ||
    2414          16 :                                                 BUNappend(old_lg->add, &b2->batCacheid, false) != GDK_SUCCEED ||
    2415             :                                                 /* 2032 is sys.args.type */
    2416           8 :                                                 BUNreplace(lg->catalog_bid, BUNfnd(lg->catalog_id, &(int){2032}), &b2->batCacheid, false) != GDK_SUCCEED) {
    2417           0 :                                                 bat_destroy(b1);
    2418           0 :                                                 bat_destroy(b2);
    2419           0 :                                                 bat_destroy(b3);
    2420           0 :                                                 return GDK_FAIL;
    2421             :                                         }
    2422           8 :                                         BBPretain(b2->batCacheid);
    2423           8 :                                         BBPretain(b2->batCacheid);
    2424           8 :                                         bat_destroy(b1);
    2425             :                                         b1 = b2;
    2426             :                                 }
    2427           8 :                                 b2 = BATconstant(0, TYPE_str, "clob", BATcount(b3), TRANSIENT);
    2428          16 :                                 if (b2 == NULL ||
    2429             :                                         /* do the update */
    2430           8 :                                         BATreplace(b1, b3, b2, false) != GDK_SUCCEED) {
    2431           0 :                                         bat_destroy(b1);
    2432           0 :                                         bat_destroy(b2);
    2433           0 :                                         bat_destroy(b3);
    2434           0 :                                         return GDK_FAIL;
    2435             :                                 }
    2436           8 :                                 bat_destroy(b2);
    2437             :                         }
    2438           8 :                         bat_destroy(b1);
    2439           8 :                         bat_destroy(b3);
    2440             :                 }
    2441             : 
    2442             :                 {
    2443             :                         /* drop STREAM TABLEs
    2444             :                          * these tables don't actually have a disk presence, only
    2445             :                          * one in the catalog (so that's why we drop them and don't
    2446             :                          * convert them); we drop them by marking the relevant rows
    2447             :                          * in various catalog tables as deleted */
    2448           8 :                         BAT *dt = temp_descriptor(logger_find_bat(lg, 2067)); /* sys._tables */
    2449           8 :                         BAT *tt = temp_descriptor(logger_find_bat(lg, 2072)); /* sys._tables.type */
    2450           8 :                         if (dt == NULL || tt == NULL) {
    2451           0 :                                 bat_destroy(dt);
    2452           0 :                                 bat_destroy(tt);
    2453           0 :                                 return GDK_FAIL;
    2454             :                         }
    2455           8 :                         BAT *cands = BATmaskedcands(0, BATcount(dt), dt, false);
    2456           8 :                         if (cands == NULL) {
    2457           0 :                                 bat_destroy(dt);
    2458           0 :                                 bat_destroy(tt);
    2459           0 :                                 return GDK_FAIL;
    2460             :                         }
    2461           8 :                         BAT *strm = BATselect(tt, cands, &(sht){4}, NULL, true, true, false);
    2462           8 :                         bat_destroy(cands);
    2463           8 :                         bat_destroy(tt);
    2464           8 :                         if (strm == NULL) {
    2465           0 :                                 bat_destroy(dt);
    2466           0 :                                 return GDK_FAIL;
    2467             :                         }
    2468           8 :                         if (strm->batCount > 0) {
    2469           0 :                                 for (BUN p = 0; p < strm->batCount; p++)
    2470           0 :                                         mskSetVal(dt, BUNtoid(strm, p), true);
    2471           0 :                                 bat_destroy(dt);
    2472           0 :                                 BAT *ids = COLnew(0, TYPE_int, 0, TRANSIENT);
    2473           0 :                                 BAT *ti = temp_descriptor(logger_find_bat(lg, 2068)); /* sys._tables.id */
    2474           0 :                                 if (ids == NULL || ti == NULL) {
    2475           0 :                                         bat_destroy(ids);
    2476           0 :                                         bat_destroy(ti);
    2477           0 :                                         bat_destroy(strm);
    2478           0 :                                         return GDK_FAIL;
    2479             :                                 }
    2480             :                                 struct {
    2481             :                                         int id, tabid, dels; /* id, table_id and deleted rows */
    2482           0 :                                 } foreign[10] = {
    2483             :                                         /* the first 7 entries are references to the table id
    2484             :                                          * the first column non-zero means the ids get collected */
    2485             :                                         {   0, 2049, 2047}, /* sys.table_partitions.table_id */
    2486             :                                         {   0, 2054, 2053}, /* sys.range_partitions.table_id */
    2487             :                                         {   0, 2060, 2059}, /* sys.value_partitions.table_id */
    2488             :                                         {2077, 2082, 2076}, /* sys._columns.table_id */
    2489             :                                         {2088, 2089, 2087}, /* sys.keys.table_id */
    2490             :                                         {2095, 2096, 2094}, /* sys.idxs.table_id */
    2491             :                                         {2100, 2102, 2099}, /* sys.triggers.table_id */
    2492             : 
    2493             :                                         /* the remaining 3 are references to collected object ids */
    2494             :                                         {   0, 2111, 2110}, /* sys.objects.id */
    2495             :                                         {   0, 2064, 2063}, /* sys.dependencies.id */
    2496             :                                         {   0, 2065, 2063}, /* sys.dependencies.depend_id */
    2497             :                                 };
    2498             : 
    2499           0 :                                 for (int i = 0; i < 10; i++) {
    2500           0 :                                         if (i == 7) {
    2501             :                                                 /* change gear: we now need to delete the
    2502             :                                                  * collected ids from sys.objects */
    2503           0 :                                                 bat_destroy(ti);
    2504             :                                                 ti = ids;
    2505             :                                                 ids = NULL;
    2506           0 :                                                 bat_destroy(strm);
    2507             :                                                 strm = NULL;
    2508             :                                         }
    2509           0 :                                         BAT *ct = temp_descriptor(logger_find_bat(lg, foreign[i].tabid));
    2510           0 :                                         BAT *dc = temp_descriptor(logger_find_bat(lg, foreign[i].dels));
    2511           0 :                                         if (ct == NULL || dc == NULL) {
    2512           0 :                                                 bat_destroy(ids);
    2513           0 :                                                 bat_destroy(strm);
    2514           0 :                                                 bat_destroy(ti);
    2515           0 :                                                 bat_destroy(ct);
    2516           0 :                                                 bat_destroy(dc);
    2517           0 :                                                 return GDK_FAIL;
    2518             :                                         }
    2519           0 :                                         cands = BATmaskedcands(0, BATcount(dc), dc, false);
    2520           0 :                                         if (cands == NULL) {
    2521           0 :                                                 bat_destroy(ids);
    2522           0 :                                                 bat_destroy(strm);
    2523           0 :                                                 bat_destroy(ti);
    2524           0 :                                                 bat_destroy(ct);
    2525           0 :                                                 bat_destroy(dc);
    2526           0 :                                                 return GDK_FAIL;
    2527             :                                         }
    2528           0 :                                         BAT *strc = BATintersect(ct, ti, cands, strm, false, false, BUN_NONE);
    2529           0 :                                         bat_destroy(cands);
    2530           0 :                                         if (strc == NULL) {
    2531           0 :                                                 bat_destroy(ids);
    2532           0 :                                                 bat_destroy(strm);
    2533           0 :                                                 bat_destroy(ti);
    2534           0 :                                                 bat_destroy(ct);
    2535           0 :                                                 bat_destroy(dc);
    2536           0 :                                                 return GDK_FAIL;
    2537             :                                         }
    2538           0 :                                         for (BUN p = 0; p < strc->batCount; p++)
    2539           0 :                                                 mskSetVal(dc, BUNtoid(strc, p), true);
    2540           0 :                                         if (foreign[i].id != 0) {
    2541           0 :                                                 BAT *ci = temp_descriptor(logger_find_bat(lg, foreign[i].id));
    2542           0 :                                                 if (ci == NULL) {
    2543           0 :                                                         bat_destroy(ids);
    2544           0 :                                                         bat_destroy(strc);
    2545           0 :                                                         bat_destroy(ct);
    2546           0 :                                                         bat_destroy(dc);
    2547           0 :                                                         bat_destroy(strm);
    2548           0 :                                                         bat_destroy(ti);
    2549           0 :                                                         return GDK_FAIL;
    2550             :                                                 }
    2551           0 :                                                 if (BATappend(ids, ci, strc, false) != GDK_SUCCEED) {
    2552           0 :                                                         bat_destroy(ids);
    2553           0 :                                                         bat_destroy(strc);
    2554           0 :                                                         bat_destroy(ct);
    2555           0 :                                                         bat_destroy(dc);
    2556           0 :                                                         bat_destroy(strm);
    2557           0 :                                                         bat_destroy(ti);
    2558           0 :                                                         bat_destroy(ci);
    2559           0 :                                                         return GDK_FAIL;
    2560             :                                                 }
    2561           0 :                                                 bat_destroy(ci);
    2562             :                                         }
    2563           0 :                                         bat_destroy(strc);
    2564           0 :                                         bat_destroy(ct);
    2565           0 :                                         bat_destroy(dc);
    2566             :                                 }
    2567           0 :                                 bat_destroy(ti);
    2568             :                         } else {
    2569           8 :                                 bat_destroy(dt);
    2570           8 :                                 bat_destroy(strm);
    2571             :                         }
    2572             :                 }
    2573             :         }
    2574             : #endif
    2575             : 
    2576             :         return GDK_SUCCEED;
    2577             : }
    2578             : 
    2579             : static int
    2580         266 : bl_create(sqlstore *store, int debug, const char *logdir, int cat_version)
    2581             : {
    2582         266 :         if (store->logger)
    2583             :                 return LOG_ERR;
    2584         266 :         store->logger = logger_create(debug, "sql", logdir, cat_version, (preversionfix_fptr)&bl_preversion, (postversionfix_fptr)&bl_postversion, store);
    2585         266 :         if (store->logger)
    2586         266 :                 return LOG_OK;
    2587             :         return LOG_ERR;
    2588             : }
    2589             : 
    2590             : static void
    2591         265 : bl_destroy(sqlstore *store)
    2592             : {
    2593         265 :         logger *l = store->logger;
    2594             : 
    2595         265 :         store->logger = NULL;
    2596         265 :         if (l)
    2597         265 :                 logger_destroy(l);
    2598         265 : }
    2599             : 
    2600             : static int
    2601        1364 : bl_flush(sqlstore *store, lng save_id)
    2602             : {
    2603        1364 :         if (store->logger)
    2604        1364 :                 return logger_flush(store->logger, save_id) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
    2605             :         return LOG_OK;
    2606             : }
    2607             : 
    2608             : static int
    2609           1 : bl_activate(sqlstore *store)
    2610             : {
    2611           1 :         if (store->logger)
    2612           1 :                 return logger_activate(store->logger) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
    2613             :         return LOG_OK;
    2614             : }
    2615             : 
    2616             : static int
    2617       14325 : bl_changes(sqlstore *store)
    2618             : {
    2619       14325 :         return (int) MIN(logger_changes(store->logger), GDK_int_max);
    2620             : }
    2621             : 
    2622             : static int
    2623         377 : bl_get_sequence(sqlstore *store, int seq, lng *id)
    2624             : {
    2625         377 :         return logger_sequence(store->logger, seq, id);
    2626             : }
    2627             : 
    2628             : static int
    2629         266 : bl_log_isnew(sqlstore *store)
    2630             : {
    2631         266 :         logger *bat_logger = store->logger;
    2632         266 :         if (BATcount(bat_logger->catalog_bid) > 10) {
    2633          79 :                 return 0;
    2634             :         }
    2635             :         return 1;
    2636             : }
    2637             : 
    2638             : static int
    2639       16954 : bl_tstart(sqlstore *store, bool flush)
    2640             : {
    2641       16954 :         return log_tstart(store->logger, flush) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
    2642             : }
    2643             : 
    2644             : static int
    2645       16954 : bl_tend(sqlstore *store)
    2646             : {
    2647       16954 :         return log_tend(store->logger) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
    2648             : }
    2649             : 
    2650             : static int
    2651       16954 : bl_tdone(sqlstore *store, ulng commit_ts)
    2652             : {
    2653       16954 :         return log_tdone(store->logger, commit_ts) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
    2654             : }
    2655             : 
    2656             : static int
    2657        6238 : bl_sequence(sqlstore *store, int seq, lng id)
    2658             : {
    2659        6238 :         return log_sequence(store->logger, seq, id) == GDK_SUCCEED ? LOG_OK : LOG_ERR;
    2660             : }
    2661             : 
    2662             : /* Write a plan entry to copy part of the given file.
    2663             :  * That part of the file must remain unchanged until the plan is executed.
    2664             :  */
    2665             : static void
    2666             : snapshot_lazy_copy_file(stream *plan, const char *name, uint64_t extent)
    2667             : {
    2668         935 :         mnstr_printf(plan, "c %" PRIu64 " %s\n", extent, name);
    2669             : }
    2670             : 
    2671             : /* Write a plan entry to write the current contents of the given file.
    2672             :  * The contents are included in the plan so the source file is allowed to
    2673             :  * change in the mean time.
    2674             :  */
    2675             : static gdk_return
    2676          10 : snapshot_immediate_copy_file(stream *plan, const char *path, const char *name)
    2677             : {
    2678             :         gdk_return ret = GDK_FAIL;
    2679             :         const size_t bufsize = 64 * 1024;
    2680             :         struct stat statbuf;
    2681             :         char *buf = NULL;
    2682             :         stream *s = NULL;
    2683             :         size_t to_copy;
    2684             : 
    2685          10 :         if (MT_stat(path, &statbuf) < 0) {
    2686           0 :                 GDKsyserror("stat failed on %s", path);
    2687           0 :                 goto end;
    2688             :         }
    2689          10 :         to_copy = (size_t) statbuf.st_size;
    2690             : 
    2691          10 :         s = open_rstream(path);
    2692          10 :         if (!s) {
    2693           0 :                 GDKerror("%s", mnstr_peek_error(NULL));
    2694           0 :                 goto end;
    2695             :         }
    2696             : 
    2697          10 :         buf = GDKmalloc(bufsize);
    2698          10 :         if (!buf) {
    2699           0 :                 GDKerror("GDKmalloc failed");
    2700           0 :                 goto end;
    2701             :         }
    2702             : 
    2703          10 :         mnstr_printf(plan, "w %zu %s\n", to_copy, name);
    2704             : 
    2705          20 :         while (to_copy > 0) {
    2706          10 :                 size_t chunk = (to_copy <= bufsize) ? to_copy : bufsize;
    2707          10 :                 ssize_t bytes_read = mnstr_read(s, buf, 1, chunk);
    2708          10 :                 if (bytes_read < 0) {
    2709           0 :                         char *err = mnstr_error(s);
    2710           0 :                         GDKerror("Reading bytes of component %s failed: %s", path, err);
    2711           0 :                         free(err);
    2712           0 :                         goto end;
    2713          10 :                 } else if (bytes_read < (ssize_t) chunk) {
    2714           0 :                         char *err = mnstr_error(s);
    2715           0 :                         GDKerror("Read only %zu/%zu bytes of component %s: %s", (size_t) bytes_read, chunk, path, err);
    2716           0 :                         free(err);
    2717           0 :                         goto end;
    2718             :                 }
    2719             : 
    2720          10 :                 ssize_t bytes_written = mnstr_write(plan, buf, 1, chunk);
    2721          10 :                 if (bytes_written < 0) {
    2722           0 :                         GDKerror("Writing to plan failed");
    2723           0 :                         goto end;
    2724          10 :                 } else if (bytes_written < (ssize_t) chunk) {
    2725           0 :                         GDKerror("write to plan truncated");
    2726           0 :                         goto end;
    2727             :                 }
    2728          10 :                 to_copy -= chunk;
    2729             :         }
    2730             : 
    2731             :         ret = GDK_SUCCEED;
    2732          10 : end:
    2733          10 :         GDKfree(buf);
    2734          10 :         if (s)
    2735          10 :                 close_stream(s);
    2736          10 :         return ret;
    2737             : }
    2738             : 
    2739             : /* Add plan entries for all relevant files in the Write Ahead Log */
    2740             : static gdk_return
    2741           5 : snapshot_wal(logger *bat_logger, stream *plan, const char *db_dir)
    2742             : {
    2743             :         char log_file[FILENAME_MAX];
    2744             :         int len;
    2745             : 
    2746           5 :         len = snprintf(log_file, sizeof(log_file), "%s/%s%s", db_dir, bat_logger->dir, LOGFILE);
    2747           5 :         if (len == -1 || (size_t)len >= sizeof(log_file)) {
    2748           0 :                 GDKerror("Could not open %s, filename is too large", log_file);
    2749           0 :                 return GDK_FAIL;
    2750             :         }
    2751           5 :         snapshot_immediate_copy_file(plan, log_file, log_file + strlen(db_dir) + 1);
    2752             : 
    2753          10 :         for (ulng id = bat_logger->saved_id+1; id <= bat_logger->id; id++) {
    2754             :                 struct stat statbuf;
    2755             : 
    2756           5 :                 len = snprintf(log_file, sizeof(log_file), "%s/%s%s." LLFMT, db_dir, bat_logger->dir, LOGFILE, id);
    2757           5 :                 if (len == -1 || (size_t)len >= sizeof(log_file)) {
    2758           0 :                         GDKerror("Could not open %s, filename is too large", log_file);
    2759           0 :                         return GDK_FAIL;
    2760             :                 }
    2761           5 :                 if (MT_stat(log_file, &statbuf) == 0) {
    2762           5 :                         snapshot_lazy_copy_file(plan, log_file + strlen(db_dir) + 1, statbuf.st_size);
    2763             :                 } else {
    2764           0 :                         GDKerror("Could not open %s", log_file);
    2765           0 :                         return GDK_FAIL;
    2766             :                 }
    2767             :         }
    2768             :         return GDK_SUCCEED;
    2769             : }
    2770             : 
    2771             : static gdk_return
    2772        1625 : snapshot_heap(stream *plan, const char *db_dir, uint64_t batid, const char *filename, const char *suffix, uint64_t extent)
    2773             : {
    2774             :         char path1[FILENAME_MAX];
    2775             :         char path2[FILENAME_MAX];
    2776        1625 :         const size_t offset = strlen(db_dir) + 1;
    2777             :         struct stat statbuf;
    2778             :         int len;
    2779             : 
    2780        1625 :         if (extent == 0) {
    2781             :                 /* nothing to copy */
    2782             :                 return GDK_SUCCEED;
    2783             :         }
    2784             :         // first check the backup dir
    2785         930 :         len = snprintf(path1, FILENAME_MAX, "%s/%s/%" PRIo64 "%s", db_dir, BAKDIR, batid, suffix);
    2786         930 :         if (len == -1 || len >= FILENAME_MAX) {
    2787           0 :                 path1[FILENAME_MAX - 1] = '\0';
    2788           0 :                 GDKerror("Could not open %s, filename is too large", path1);
    2789           0 :                 return GDK_FAIL;
    2790             :         }
    2791         930 :         if (MT_stat(path1, &statbuf) == 0) {
    2792           0 :                 snapshot_lazy_copy_file(plan, path1 + offset, extent);
    2793           0 :                 return GDK_SUCCEED;
    2794             :         }
    2795         930 :         if (errno != ENOENT) {
    2796           0 :                 GDKsyserror("Error stat'ing %s", path1);
    2797             :                 return GDK_FAIL;
    2798             :         }
    2799             : 
    2800             :         // then check the regular location
    2801         930 :         len = snprintf(path2, FILENAME_MAX, "%s/%s/%s%s", db_dir, BATDIR, filename, suffix);
    2802         930 :         if (len == -1 || len >= FILENAME_MAX) {
    2803           0 :                 path2[FILENAME_MAX - 1] = '\0';
    2804           0 :                 GDKerror("Could not open %s, filename is too large", path2);
    2805           0 :                 return GDK_FAIL;
    2806             :         }
    2807         930 :         if (MT_stat(path2, &statbuf) == 0) {
    2808         930 :                 snapshot_lazy_copy_file(plan, path2 + offset, extent);
    2809         930 :                 return GDK_SUCCEED;
    2810             :         }
    2811           0 :         if (errno != ENOENT) {
    2812           0 :                 GDKsyserror("Error stat'ing %s", path2);
    2813             :                 return GDK_FAIL;
    2814             :         }
    2815             : 
    2816           0 :         GDKerror("One of %s and %s must exist", path1, path2);
    2817           0 :         return GDK_FAIL;
    2818             : }
    2819             : 
    2820             : /* Add plan entries for all persistent BATs by looping over the BBP.dir.
    2821             :  * Also include the BBP.dir itself.
    2822             :  */
    2823             : static gdk_return
    2824           5 : snapshot_bats(stream *plan, const char *db_dir)
    2825             : {
    2826             :         char bbpdir[FILENAME_MAX];
    2827             :         stream *cat = NULL;
    2828             :         char line[1024];
    2829             :         int gdk_version, len;
    2830             :         gdk_return ret = GDK_FAIL;
    2831             : 
    2832           5 :         len = snprintf(bbpdir, FILENAME_MAX, "%s/%s/%s", db_dir, BAKDIR, "BBP.dir");
    2833           5 :         if (len == -1 || len >= FILENAME_MAX) {
    2834           0 :                 GDKerror("Could not open %s, filename is too large", bbpdir);
    2835           0 :                 goto end;
    2836             :         }
    2837           5 :         ret = snapshot_immediate_copy_file(plan, bbpdir, bbpdir + strlen(db_dir) + 1);
    2838           5 :         if (ret != GDK_SUCCEED)
    2839           0 :                 goto end;
    2840             : 
    2841             :         // Open the catalog and parse the header
    2842           5 :         cat = open_rastream(bbpdir);
    2843           5 :         if (cat == NULL) {
    2844           0 :                 GDKerror("Could not open %s for reading: %s", bbpdir, mnstr_peek_error(NULL));
    2845           0 :                 goto end;
    2846             :         }
    2847           5 :         if (mnstr_readline(cat, line, sizeof(line)) < 0) {
    2848           0 :                 GDKerror("Could not read first line of %s", bbpdir);
    2849           0 :                 goto end;
    2850             :         }
    2851           5 :         if (sscanf(line, "BBP.dir, GDKversion %d", &gdk_version) != 1) {
    2852           0 :                 GDKerror("Invalid first line of %s", bbpdir);
    2853           0 :                 goto end;
    2854             :         }
    2855           5 :         if (gdk_version != 061045U) {
    2856             :                 // If this version number has changed, the structure of BBP.dir
    2857             :                 // may have changed. Update this whole function to take this
    2858             :                 // into account.
    2859             :                 // Note: when startup has completed BBP.dir is guaranteed
    2860             :                 // to the latest format so we don't have to support any older
    2861             :                 // formats in this function.
    2862           0 :                 GDKerror("GDK version mismatch in snapshot yet");
    2863           0 :                 goto end;
    2864             :         }
    2865           5 :         if (mnstr_readline(cat, line, sizeof(line)) < 0) {
    2866           0 :                 GDKerror("Couldn't skip the second line of %s", bbpdir);
    2867           0 :                 goto end;
    2868             :         }
    2869           5 :         if (mnstr_readline(cat, line, sizeof(line)) < 0) {
    2870           0 :                 GDKerror("Couldn't skip the third line of %s", bbpdir);
    2871           0 :                 goto end;
    2872             :         }
    2873             : 
    2874             :         /* TODO get transaction id and last processed log file id */
    2875           5 :         if (mnstr_readline(cat, line, sizeof(line)) < 0) {
    2876           0 :                 GDKerror("Couldn't skip the 4th line of %s", bbpdir);
    2877           0 :                 goto end;
    2878             :         }
    2879             : 
    2880        1285 :         while (mnstr_readline(cat, line, sizeof(line)) > 0) {
    2881             :                 uint64_t batid;
    2882             :                 char type[16];
    2883             :                 uint16_t width;
    2884             :                 uint64_t tail_free;
    2885             :                 uint64_t theap_free;
    2886             :                 char filename[sizeof(BBP_physical(0))];
    2887             :                 // The lines in BBP.dir come in various lengths.
    2888             :                 // we try to parse the longest variant then check
    2889             :                 // the return value of sscanf to see which fields
    2890             :                 // were actually present.
    2891        1280 :                 int scanned = sscanf(line,
    2892             :                                 // Taken from the sscanf in BBPreadEntries() in gdk_bbp.c.
    2893             :                                 // 8 fields, we need field 1 (batid) and field 4 (filename)
    2894             :                                 "%" SCNu64 " %*s %*s %19s %*s %*s %*s %*s"
    2895             : 
    2896             :                                 // Taken from the sscanf in heapinit() in gdk_bbp.c.
    2897             :                                 // 14 fields, we need fields 1 (type), 2 (width), 10 (free)
    2898             :                                 " %10s %" SCNu16 " %*s %*s %*s %*s %*s %*s %*s %" SCNu64 " %*s %*s %*s %*s"
    2899             : 
    2900             :                                 // Taken from the sscanf in vheapinit() in gdk_bbp.c.
    2901             :                                 // 3 fields, we need field 1 (free).
    2902             :                                 "%" SCNu64 " %*s ^*s"
    2903             :                                 ,
    2904             :                                 &batid, filename,
    2905             :                                 type, &width, &tail_free,
    2906             :                                 &theap_free);
    2907             : 
    2908             :                 // The following switch uses fallthroughs to make
    2909             :                 // the larger cases include the work of the smaller cases.
    2910        1280 :                 switch (scanned) {
    2911           0 :                         default:
    2912           0 :                                 GDKerror("Couldn't parse (%d) %s line: %s", scanned, bbpdir, line);
    2913           0 :                                 goto end;
    2914         345 :                         case 6:
    2915             :                                 // tail and theap
    2916         345 :                                 ret = snapshot_heap(plan, db_dir, batid, filename, ".theap", theap_free);
    2917         345 :                                 if (ret != GDK_SUCCEED)
    2918           0 :                                         goto end;
    2919             :                                 /* fallthrough */
    2920             :                         case 5:
    2921             :                                 // tail only
    2922        1280 :                                 snapshot_heap(plan, db_dir, batid, filename,
    2923        1280 :                                                           strcmp(type, "str") == 0 ? width == 1 ? ".tail1" : width == 2 ? ".tail2" :
    2924             : #if SIZEOF_VAR_T == 8
    2925             :                                                           width == 4 ? ".tail4" :
    2926             : #endif
    2927         345 :                                                           ".tail" : ".tail",
    2928             :                                                           tail_free);
    2929        1280 :                                 if (ret != GDK_SUCCEED)
    2930           0 :                                         goto end;
    2931             :                                 /* fallthrough */
    2932             :                         case 2:
    2933             :                                 // no tail?
    2934             :                                 break;
    2935             :                 }
    2936             :         }
    2937             : 
    2938           5 : end:
    2939           5 :         if (cat) {
    2940           5 :                 close_stream(cat);
    2941             :         }
    2942           5 :         return ret;
    2943             : }
    2944             : 
    2945             : /* Add a file to the plan which records the current wlc status, if any.
    2946             :  * In particular, `wlc_batches`.
    2947             :  *
    2948             :  * With this information, a replica initialized from this snapshot can
    2949             :  * be configured to catch up with its master by replaying later transactions.
    2950             :  */
    2951             : static gdk_return
    2952           5 : snapshot_wlc(stream *plan, const char *db_dir)
    2953             : {
    2954           5 :         const char name[] = "wlr.config.in";
    2955             :         char buf[1024];
    2956             :         int len;
    2957             : 
    2958             :         (void)db_dir;
    2959             : 
    2960           5 :         if (wlc_state != WLC_RUN)
    2961             :                 return GDK_SUCCEED;
    2962             : 
    2963           0 :         len = snprintf(buf, sizeof(buf),
    2964             :                 "beat=%d\n"
    2965             :                 "batches=%d\n"
    2966             :                 , wlc_beat, wlc_batches
    2967             :         );
    2968             : 
    2969           0 :         mnstr_printf(plan, "w %d %s\n", len, name);
    2970           0 :         mnstr_write(plan, buf, 1, len);
    2971             : 
    2972           0 :         return GDK_SUCCEED;
    2973             : }
    2974             : 
    2975             : static gdk_return
    2976           5 : snapshot_vaultkey(stream *plan, const char *db_dir)
    2977             : {
    2978             :         char path[FILENAME_MAX];
    2979             :         struct stat statbuf;
    2980             : 
    2981           5 :         int len = snprintf(path, FILENAME_MAX, "%s/.vaultkey", db_dir);
    2982           5 :         if (len == -1 || len >= FILENAME_MAX) {
    2983           0 :                 path[FILENAME_MAX - 1] = '\0';
    2984           0 :                 GDKerror("Could not open %s, filename is too large", path);
    2985           0 :                 return GDK_FAIL;
    2986             :         }
    2987           5 :         if (MT_stat(path, &statbuf) == 0) {
    2988           0 :                 snapshot_lazy_copy_file(plan, ".vaultkey", statbuf.st_size);
    2989           0 :                 return GDK_SUCCEED;
    2990             :         }
    2991           5 :         if (errno == ENOENT) {
    2992             :                 // No .vaultkey? Fine.
    2993             :                 return GDK_SUCCEED;
    2994             :         }
    2995             : 
    2996           0 :         GDKsyserror("Error stat'ing %s", path);
    2997           0 :         return GDK_FAIL;
    2998             : }
    2999             : static gdk_return
    3000           5 : bl_snapshot(sqlstore *store, stream *plan)
    3001             : {
    3002           5 :         logger *bat_logger = store->logger;
    3003             :         gdk_return ret;
    3004             :         char *db_dir = NULL;
    3005             :         size_t db_dir_len;
    3006             : 
    3007             :         // Farm 0 is always the persistent farm.
    3008           5 :         db_dir = GDKfilepath(0, NULL, "", NULL);
    3009           5 :         db_dir_len = strlen(db_dir);
    3010           5 :         if (db_dir[db_dir_len - 1] == DIR_SEP)
    3011           5 :                 db_dir[db_dir_len - 1] = '\0';
    3012             : 
    3013           5 :         mnstr_printf(plan, "%s\n", db_dir);
    3014             : 
    3015             :         // Please monetdbd
    3016           5 :         mnstr_printf(plan, "w 0 .uplog\n");
    3017             : 
    3018           5 :         ret = snapshot_vaultkey(plan, db_dir);
    3019           5 :         if (ret != GDK_SUCCEED)
    3020           0 :                 goto end;
    3021             : 
    3022           5 :         ret = snapshot_bats(plan, db_dir);
    3023           5 :         if (ret != GDK_SUCCEED)
    3024           0 :                 goto end;
    3025             : 
    3026           5 :         ret = snapshot_wal(bat_logger, plan, db_dir);
    3027           5 :         if (ret != GDK_SUCCEED)
    3028           0 :                 goto end;
    3029             : 
    3030           5 :         ret = snapshot_wlc(plan, db_dir);
    3031           5 :         if (ret != GDK_SUCCEED)
    3032           0 :                 goto end;
    3033             : 
    3034             :         ret = GDK_SUCCEED;
    3035           5 : end:
    3036             :         if (db_dir)
    3037           5 :                 GDKfree(db_dir);
    3038           5 :         return ret;
    3039             : }
    3040             : 
    3041             : void
    3042         266 : bat_logger_init( logger_functions *lf )
    3043             : {
    3044         266 :         lf->create = bl_create;
    3045         266 :         lf->destroy = bl_destroy;
    3046         266 :         lf->flush = bl_flush;
    3047         266 :         lf->activate = bl_activate;
    3048         266 :         lf->changes = bl_changes;
    3049         266 :         lf->get_sequence = bl_get_sequence;
    3050         266 :         lf->log_isnew = bl_log_isnew;
    3051         266 :         lf->log_tstart = bl_tstart;
    3052         266 :         lf->log_tend = bl_tend;
    3053         266 :         lf->log_tdone = bl_tdone;
    3054         266 :         lf->log_sequence = bl_sequence;
    3055         266 :         lf->get_snapshot_files = bl_snapshot;
    3056         266 : }

Generated by: LCOV version 1.14