LCOV - code coverage report
Current view: top level - sql/backends/monet5/vaults/fits - fits.c (source / functions) Hit Total Coverage
Test: coverage.info Lines: 2 675 0.3 %
Date: 2021-10-13 02:24:04 Functions: 1 10 10.0 %

          Line data    Source code
       1             : /*
       2             :  * This Source Code Form is subject to the terms of the Mozilla Public
       3             :  * License, v. 2.0.  If a copy of the MPL was not distributed with this
       4             :  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
       5             :  *
       6             :  * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V.
       7             :  */
       8             : 
       9             : /*
      10             :  * Authors: M. Ivanova, M. Kersten, N. Nes
      11             :  *
      12             :  * This module contains primitives for accessing data in FITS file format.
      13             :  */
      14             : 
      15             : #include "monetdb_config.h"
      16             : #include <glob.h>
      17             : 
      18             : /* clash with GDK? */
      19             : #undef ttype
      20             : #include <fitsio.h>
      21             : #include <fitsio2.h>
      22             : #include <longnam.h>
      23             : 
      24             : #include "fits.h"
      25             : #include "mutils.h"
      26             : #include "sql_mvc.h"
      27             : #include "sql_scenario.h"
      28             : #include "sql_execute.h"
      29             : #include "sql.h"
      30             : #include "mal_exception.h"
      31             : 
      32             : #define FITS_INS_COL "INSERT INTO sys.fits_columns(id, name, type, units, number, table_id) \
      33             :          VALUES(%d,'%s','%s','%s',%d,%d);"
      34             : #define ATTACHDIR "call sys.fitsattach('%s');"
      35             : 
      36             : static void
      37           0 : FITSinitCatalog(mvc *m)
      38             : {
      39             :         sql_schema *sch;
      40             :         sql_table *fits_tp, *fits_fl, *fits_tbl, *fits_col;
      41           0 :         sql_column *col = NULL;
      42             : 
      43           0 :         sch = mvc_bind_schema(m, "sys");
      44             : 
      45           0 :         fits_fl = mvc_bind_table(m, sch, "fits_files");
      46           0 :         if (fits_fl == NULL) {
      47           0 :                 mvc_create_table(&fits_fl, m, sch, "fits_files", tt_table, 0, SQL_PERSIST, 0, 2, 0);
      48           0 :                 mvc_create_column_(&col, m, fits_fl, "id", "int", 32);
      49           0 :                 mvc_create_column_(&col, m, fits_fl, "name", "varchar", 80);
      50             :         }
      51             : 
      52           0 :         fits_tbl = mvc_bind_table(m, sch, "fits_tables");
      53           0 :         if (fits_tbl == NULL) {
      54           0 :                 mvc_create_table(&fits_tbl, m, sch, "fits_tables", tt_table, 0, SQL_PERSIST, 0, 8, 0);
      55           0 :                 mvc_create_column_(&col, m, fits_tbl, "id", "int", 32);
      56           0 :                 mvc_create_column_(&col, m, fits_tbl, "name", "varchar", 80);
      57           0 :                 mvc_create_column_(&col, m, fits_tbl, "columns", "int", 32);
      58           0 :                 mvc_create_column_(&col, m, fits_tbl, "file_id", "int", 32);
      59           0 :                 mvc_create_column_(&col, m, fits_tbl, "hdu", "int", 32);
      60           0 :                 mvc_create_column_(&col, m, fits_tbl, "date", "varchar", 80);
      61           0 :                 mvc_create_column_(&col, m, fits_tbl, "origin", "varchar", 80);
      62           0 :                 mvc_create_column_(&col, m, fits_tbl, "comment", "varchar", 80);
      63             :         }
      64             : 
      65           0 :         fits_col = mvc_bind_table(m, sch, "fits_columns");
      66           0 :         if (fits_col == NULL) {
      67           0 :                 mvc_create_table(&fits_col, m, sch, "fits_columns", tt_table, 0, SQL_PERSIST, 0, 6, 0);
      68           0 :                 mvc_create_column_(&col, m, fits_col, "id", "int", 32);
      69           0 :                 mvc_create_column_(&col, m, fits_col, "name", "varchar", 80);
      70           0 :                 mvc_create_column_(&col, m, fits_col, "type", "varchar", 80);
      71           0 :                 mvc_create_column_(&col, m, fits_col, "units", "varchar", 80);
      72           0 :                 mvc_create_column_(&col, m, fits_col, "number", "int", 32);
      73           0 :                 mvc_create_column_(&col, m, fits_col, "table_id", "int", 32);
      74             :         }
      75             : 
      76           0 :         fits_tp = mvc_bind_table(m, sch, "fits_table_properties");
      77           0 :         if (fits_tp == NULL) {
      78           0 :                 mvc_create_table(&fits_tp, m, sch, "fits_table_properties", tt_table, 0, SQL_PERSIST, 0, 5, 0);
      79           0 :                 mvc_create_column_(&col, m, fits_tp, "table_id", "int", 32);
      80           0 :                 mvc_create_column_(&col, m, fits_tp, "xtension", "varchar", 80);
      81           0 :                 mvc_create_column_(&col, m, fits_tp, "bitpix", "int", 32);
      82           0 :                 mvc_create_column_(&col, m, fits_tp, "stilvers", "varchar", 80);
      83           0 :                 mvc_create_column_(&col, m, fits_tp, "stilclas", "varchar", 80);
      84             :         }
      85           0 : }
      86             : 
      87             : static int
      88           0 : fits2mtype(int t, int rep)
      89             : {
      90           0 :         if (rep > 1) {
      91           0 :                 return TYPE_blob;
      92             :         }
      93           0 :         switch (t) {
      94             :         case TBIT:
      95             :         case TLOGICAL:
      96             :                 return TYPE_bit;
      97           0 :         case TBYTE:
      98             :         case TSBYTE:
      99           0 :                 return TYPE_bte;
     100           0 :         case TSTRING:
     101           0 :                 return TYPE_str;
     102           0 :         case TUSHORT:
     103             :         case TSHORT:
     104           0 :                 return TYPE_sht;
     105           0 :         case TUINT:
     106             :         case TINT:
     107           0 :                 return TYPE_int;
     108           0 :         case TLONG:
     109             :         case TULONG:
     110             :         case TLONGLONG:
     111           0 :                 return TYPE_lng;
     112           0 :         case TFLOAT:
     113           0 :                 return TYPE_flt;
     114           0 :         case TDOUBLE:
     115           0 :                 return TYPE_dbl;
     116             :         /* missing */
     117           0 :         case TCOMPLEX:
     118             :         case TDBLCOMPLEX:
     119           0 :                 return -1;
     120             :         }
     121           0 :         return -1;
     122             : }
     123             : 
     124             : static int
     125           0 : fits2subtype(sql_subtype *tpe, int t, long rep, long wid) /* type long used by fits library */
     126             : {
     127           0 :         if (rep > 1) {
     128           0 :                 sql_find_subtype(tpe, "blob", (unsigned int)rep*wid, 0);
     129           0 :                 return 1;
     130             :         }
     131           0 :         switch (t) {
     132           0 :         case TBIT:
     133             :         case TLOGICAL:
     134           0 :                 sql_find_subtype(tpe, "boolean", 0, 0);
     135           0 :                 break;
     136           0 :         case TBYTE:
     137             :         case TSBYTE:
     138           0 :                 sql_find_subtype(tpe, "char", 1, 0);
     139           0 :                 break;
     140           0 :         case TSTRING:
     141           0 :                 sql_find_subtype(tpe, "varchar", (unsigned int)wid, 0);
     142           0 :                 break;
     143           0 :         case TUSHORT:
     144             :         case TSHORT:
     145           0 :                 sql_find_subtype(tpe, "smallint", 16, 0);
     146           0 :                 break;
     147           0 :         case TUINT:
     148             :         case TINT:
     149           0 :                 sql_find_subtype(tpe, "int", 32, 0);
     150           0 :                 break;
     151           0 :         case TULONG:
     152             :         case TLONG:
     153             :         case TLONGLONG:
     154           0 :                 sql_find_subtype(tpe, "bigint", 64, 0);
     155           0 :                 break;
     156           0 :         case TFLOAT:
     157           0 :                 sql_find_subtype(tpe, "real", 32, 0);
     158           0 :                 break;
     159           0 :         case TDOUBLE:
     160           0 :                 sql_find_subtype(tpe, "double", 51, 0);
     161           0 :                 break;
     162             :         /* missing */
     163             :         case TCOMPLEX:
     164             :         case TDBLCOMPLEX:
     165             :                 return -1;
     166             :         }
     167             :         return 1;
     168             : }
     169             : 
     170           0 : str FITSexportTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
     171             : {
     172             :         str msg = MAL_SUCCEED;
     173           0 :         str tname = *getArgReference_str(stk, pci, 1);
     174           0 :         mvc *m = NULL;
     175             :         sql_trans *tr;
     176             :         sql_schema *sch;
     177             :         sql_table *tbl, *column, *tables = NULL;
     178             :         sql_column *col;
     179             :         oid rid = oid_nil;
     180             :         str type, name, *colname, *tform;
     181             :         fitsfile *fptr;
     182             :         char filename[BUFSIZ];
     183             :         size_t nrows = 0;
     184             :         long optimal; /* type long used by fits library */
     185             :         rids * rs;
     186             : 
     187             :         int tm0, texportboolean=0, texportchar=0, texportstring=0, texportshort=0, texportint=0, texportlng=0, texportfloat=0, texportdouble=0;
     188             :         size_t numberrow = 0, dimension = 0;
     189           0 :         int cc = 0, status = 0, j = 0, columns, *fid, block = 0;
     190             :         int boolcols = 0, charcols = 0, strcols = 0, shortcols = 0, intcols = 0, lngcols = 0, floatcols = 0, doublecols = 0;
     191             :         int hdutype;
     192             : 
     193             :         char *charvalue, *readcharrows;
     194             :         str strvalue; char **readstrrows;
     195             :         short *shortvalue, *readshortrows;
     196             :         int *intvalue, *readintrows;
     197             :         lng *lngvalue, *readlngrows;
     198             :         float *realvalue, *readfloatrows;
     199             :         double *doublevalue, *readdoublerows;
     200             :         _Bool *boolvalue, *readboolrows;
     201             : 
     202           0 :         if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
     203             :                 return msg;
     204           0 :         if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
     205             :                 return msg;
     206             : 
     207           0 :         tr = m->session->tr;
     208           0 :         sqlstore *store = tr->store;
     209           0 :         sch = mvc_bind_schema(m, "sys");
     210             : 
     211             :         /* First step: look if the table exists in the database. If the table is not in the database, the export function cannot continue */
     212             : 
     213           0 :         tbl = mvc_bind_table(m, sch, tname);
     214           0 :         if (tbl == NULL) {
     215           0 :                 msg = createException (MAL, "fits.exporttable", SQLSTATE(FI000) "Table %s is missing.\n", tname);
     216           0 :                 return msg;
     217             :         }
     218             : 
     219             : 
     220           0 :         columns = ol_length((*tbl).columns);
     221           0 :         colname = (str *) GDKmalloc(columns * sizeof(str));
     222           0 :         tform = (str *) GDKmalloc(columns * sizeof(str));
     223           0 :         if (colname == NULL || tform == NULL) {
     224           0 :                 GDKfree(colname);
     225           0 :                 GDKfree(tform);
     226           0 :                 throw(MAL, "fits.exporttable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     227             :         }
     228             : 
     229           0 :         TRC_DEBUG(FITS, "Number of columns: %d\n", columns);
     230             : 
     231           0 :         tables = mvc_bind_table(m, sch, "_tables");
     232           0 :         col = mvc_bind_column(m, tables, "name");
     233           0 :         rid = store->table_api.column_find_row(m->session->tr, col, tname, NULL);
     234             : 
     235           0 :         col = mvc_bind_column(m, tables, "id");
     236           0 :         fid = (int*) store->table_api.column_find_value(m->session->tr, col, rid);
     237             : 
     238           0 :         column =  mvc_bind_table(m, sch, "_columns");
     239           0 :         col = mvc_bind_column(m, column, "table_id");
     240             : 
     241           0 :         rs = store->table_api.rids_select(m->session->tr, col, (void *) fid, (void *) fid, NULL);
     242           0 :         GDKfree(fid);
     243             : 
     244           0 :         while ((rid = store->table_api.rids_next(rs)), !is_oid_nil(rid))
     245             :         {
     246           0 :                 col = mvc_bind_column(m, column, "name");
     247           0 :                 name = (char *) store->table_api.column_find_value(m->session->tr, col, rid);
     248           0 :                 colname[j] = toLower(name);
     249           0 :                 GDKfree(name);
     250             : 
     251           0 :                 col = mvc_bind_column(m, column, "type");
     252           0 :                 type = (char *) store->table_api.column_find_value(m->session->tr, col, rid);
     253             : 
     254           0 :                 if (strcmp(type,"boolean")==0) tform[j] = "1L";
     255             : 
     256           0 :                 if (strcmp(type,"char")==0) tform[j] = "1S";
     257             : 
     258           0 :                 if (strcmp(type,"varchar")==0) tform[j] = "8A";
     259             : 
     260           0 :                 if (strcmp(type,"smallint")==0) tform[j] = "1I";
     261             : 
     262           0 :                 if (strcmp(type,"int")==0) tform[j] = "1J";
     263             : 
     264           0 :                 if (strcmp(type,"bigint")==0) tform[j] = "1K";
     265             : 
     266           0 :                 if (strcmp(type,"real")==0) tform[j] = "1E";
     267             : 
     268           0 :                 if (strcmp(type,"double")==0) tform[j] = "1D";
     269           0 :                 GDKfree(type);
     270             : 
     271           0 :                 j++;
     272             :         }
     273             : 
     274           0 :         col = mvc_bind_column(m, tbl, colname[0]);
     275             : 
     276           0 :         nrows = store->storage_api.count_col(tr, col, 0);
     277           0 :         assert(nrows <= (size_t) GDK_oid_max);
     278             : 
     279           0 :         snprintf(filename,BUFSIZ,"\n%s.fit",tname);
     280           0 :         TRC_INFO(FITS, "Filename: %s\n", filename);
     281             : 
     282             :         MT_remove(filename);
     283             : 
     284           0 :         status=0;
     285             : 
     286           0 :         fits_create_file(&fptr, filename, &status);
     287           0 :         fits_create_img(fptr,  USHORT_IMG, 0, NULL, &status);
     288           0 :         fits_close_file(fptr, &status);
     289           0 :         fits_open_file(&fptr, filename, READWRITE, &status);
     290             : 
     291           0 :         fits_movabs_hdu(fptr, 1, &hdutype, &status);
     292           0 :         fits_create_tbl( fptr, BINARY_TBL, 0, columns, colname, tform, NULL, tname, &status);
     293             : 
     294           0 :         for (cc = 0; cc < columns; cc++)
     295             :         {
     296             :                 char * columntype;
     297           0 :                 col = mvc_bind_column(m, tbl, colname[cc]);
     298           0 :                 columntype = col -> type.type->base.name;
     299             : 
     300           0 :                 if (strcmp(columntype,"boolean")==0)
     301             :                 {
     302           0 :                         boolcols++; dimension = 0; block = 0;
     303           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     304           0 :                         readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal);
     305             : 
     306           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     307             :                         {
     308           0 :                                 boolvalue = (_Bool*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     309           0 :                                 readboolrows[dimension] = *boolvalue;
     310           0 :                                 GDKfree(boolvalue);
     311           0 :                                 dimension++;
     312             : 
     313           0 :                                 if (dimension == (size_t) optimal)
     314             :                                 {
     315             :                                         dimension = 0;
     316           0 :                                         tm0 = GDKms();
     317           0 :                                         fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, optimal, readboolrows, &status);
     318           0 :                                         texportboolean += GDKms() - tm0;
     319           0 :                                         GDKfree(readboolrows);
     320           0 :                                         readboolrows = (_Bool *) GDKmalloc (sizeof(_Bool) * optimal);
     321           0 :                                         block++;
     322             :                                 }
     323             :                         }
     324           0 :                         tm0 = GDKms();
     325           0 :                         fits_write_col(fptr, TLOGICAL, cc+1, (optimal*block)+1, 1, dimension, readboolrows, &status);
     326           0 :                         texportboolean += GDKms() - tm0;
     327           0 :                         GDKfree(readboolrows);
     328             :                 }
     329             : 
     330           0 :                 if (strcmp(columntype,"char")==0)
     331             :                 {
     332           0 :                         charcols++; dimension = 0; block = 0;
     333           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     334           0 :                         readcharrows = (char *) GDKmalloc (sizeof(char) * optimal);
     335             : 
     336           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     337             :                         {
     338           0 :                                 charvalue = (char*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     339           0 :                                 readcharrows[dimension] = *charvalue;
     340           0 :                                 GDKfree(charvalue);
     341           0 :                                 dimension++;
     342             : 
     343           0 :                                 if (dimension == (size_t) optimal)
     344             :                                 {
     345             :                                         dimension = 0;
     346           0 :                                         tm0 = GDKms();
     347           0 :                                         fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, optimal, readcharrows, &status);
     348           0 :                                         texportchar += GDKms() - tm0;
     349           0 :                                         GDKfree(readcharrows);
     350           0 :                                         readcharrows = (char *) GDKmalloc (sizeof(char) * optimal);
     351           0 :                                         block++;
     352             :                                 }
     353             :                         }
     354           0 :                         tm0 = GDKms();
     355           0 :                         fits_write_col(fptr, TBYTE, cc+1, (optimal*block)+1, 1, dimension, readcharrows, &status);
     356           0 :                         texportchar += GDKms() - tm0;
     357           0 :                         GDKfree(readcharrows);
     358             :                 }
     359             : 
     360           0 :                 if (strcmp(columntype,"varchar")==0)
     361             :                 {
     362           0 :                         strcols++; dimension=0; block=0;
     363           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     364           0 :                         readstrrows = (char **) GDKmalloc (sizeof(char *) * optimal);
     365             : 
     366           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     367             :                         {
     368           0 :                                 strvalue = (char *) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     369           0 :                                 readstrrows[dimension] = strvalue;
     370           0 :                                 dimension++;
     371             : 
     372           0 :                                 if (dimension == (size_t) optimal)
     373             :                                 {
     374             :                                         dimension = 0;
     375           0 :                                         tm0 = GDKms();
     376           0 :                                         fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, optimal, readstrrows, &status);
     377           0 :                                         texportstring += GDKms() - tm0;
     378           0 :                                         for (dimension = 0; dimension < (size_t) optimal; dimension++)
     379           0 :                                                 GDKfree(readstrrows[dimension]);
     380             :                                         dimension = 0;
     381           0 :                                         GDKfree(readstrrows);
     382           0 :                                         readstrrows = (char **) GDKmalloc(sizeof(char *) * optimal);
     383           0 :                                         block++;
     384             :                                 }
     385             :                         }
     386           0 :                         tm0 = GDKms();
     387           0 :                         fits_write_col_str(fptr, cc+1, (optimal*block)+1, 1, dimension, readstrrows, &status);
     388           0 :                         texportstring += GDKms() - tm0;
     389           0 :                         for (numberrow = 0; numberrow < dimension; numberrow++)
     390           0 :                                 GDKfree(readstrrows[numberrow]);
     391           0 :                         GDKfree(readstrrows);
     392             :                 }
     393             : 
     394           0 :                 if (strcmp(columntype,"smallint")==0)
     395             :                 {
     396           0 :                         shortcols++; dimension = 0; block = 0;
     397           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     398           0 :                         readshortrows = (short *) GDKmalloc (sizeof(short) * optimal);
     399             : 
     400           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     401             :                         {
     402           0 :                                 shortvalue = (short*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     403           0 :                                 readshortrows[dimension] = *shortvalue;
     404           0 :                                 GDKfree(shortvalue);
     405           0 :                                 dimension++;
     406             : 
     407           0 :                                 if (dimension == (size_t) optimal)
     408             :                                 {
     409             :                                         dimension = 0;
     410           0 :                                         tm0 = GDKms();
     411           0 :                                         fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, optimal, readshortrows, &status);
     412           0 :                                         texportshort += GDKms() - tm0;
     413           0 :                                         GDKfree(readshortrows);
     414           0 :                                         readshortrows = (short *) GDKmalloc (sizeof(short) * optimal);
     415           0 :                                         block++;
     416             :                                 }
     417             :                         }
     418           0 :                         tm0 = GDKms();
     419           0 :                         fits_write_col(fptr, TSHORT, cc+1, (optimal*block)+1, 1, dimension, readshortrows, &status);
     420           0 :                         texportshort += GDKms() - tm0;
     421           0 :                         GDKfree(readshortrows);
     422             :                 }
     423             : 
     424           0 :                 if (strcmp(columntype,"int")==0)
     425             :                 {
     426           0 :                         intcols++; dimension = 0; block = 0;
     427           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     428           0 :                         readintrows = (int *) GDKmalloc (sizeof(int) * optimal);
     429             : 
     430           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     431             :                         {
     432           0 :                                 intvalue = (int*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     433           0 :                                 readintrows[dimension] = *intvalue;
     434           0 :                                 GDKfree(intvalue);
     435           0 :                                 dimension++;
     436             : 
     437           0 :                                 if (dimension == (size_t) optimal)
     438             :                                 {
     439             :                                         dimension = 0;
     440           0 :                                         tm0 = GDKms();
     441           0 :                                         fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, optimal, readintrows, &status);
     442           0 :                                         texportint += GDKms() - tm0;
     443           0 :                                         GDKfree(readintrows);
     444           0 :                                         readintrows = (int *) GDKmalloc (sizeof(int) * optimal);
     445           0 :                                         block++;
     446             :                                 }
     447             :                         }
     448           0 :                         tm0 = GDKms();
     449           0 :                         fits_write_col(fptr, TINT, cc+1, (optimal*block)+1, 1, dimension, readintrows, &status);
     450           0 :                         texportint += GDKms() - tm0;
     451           0 :                         GDKfree(readintrows);
     452             :                 }
     453             : 
     454           0 :                 if (strcmp(columntype,"bigint")==0)
     455             :                 {
     456           0 :                         lngcols++; dimension = 0; block = 0;
     457           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     458           0 :                         readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal);
     459             : 
     460           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     461             :                         {
     462           0 :                                 lngvalue = (lng*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     463           0 :                                 readlngrows[dimension] = *lngvalue;
     464           0 :                                 GDKfree(lngvalue);
     465           0 :                                 dimension++;
     466             : 
     467           0 :                                 if (dimension == (size_t) optimal)
     468             :                                 {
     469             :                                         dimension = 0;
     470           0 :                                         tm0 = GDKms();
     471           0 :                                         fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, optimal, readlngrows, &status);
     472           0 :                                         texportlng += GDKms() - tm0;
     473           0 :                                         GDKfree(readlngrows);
     474           0 :                                         readlngrows = (lng *) GDKmalloc (sizeof(lng) * optimal);
     475           0 :                                         block++;
     476             :                                 }
     477             :                         }
     478           0 :                         tm0 = GDKms();
     479           0 :                         fits_write_col(fptr, TLONG, cc+1, (optimal*block)+1, 1, dimension, readlngrows, &status);
     480           0 :                         texportlng += GDKms() - tm0;
     481           0 :                         GDKfree(readlngrows);
     482             :                 }
     483             : 
     484           0 :                 if (strcmp(columntype,"real")==0)
     485             :                 {
     486           0 :                         floatcols++; dimension = 0; block = 0;
     487           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     488           0 :                         readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal);
     489             : 
     490           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     491             :                         {
     492           0 :                                 realvalue = (float*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     493           0 :                                 readfloatrows[dimension] = *realvalue;
     494           0 :                                 GDKfree(realvalue);
     495           0 :                                 dimension++;
     496             : 
     497           0 :                                 if (dimension == (size_t) optimal)
     498             :                                 {
     499             :                                         dimension = 0;
     500           0 :                                         tm0 = GDKms();
     501           0 :                                         fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, optimal, readfloatrows, &status);
     502           0 :                                         texportfloat += GDKms() - tm0;
     503           0 :                                         GDKfree(readfloatrows);
     504           0 :                                         readfloatrows = (float *) GDKmalloc (sizeof(float) * optimal);
     505           0 :                                         block++;
     506             :                                 }
     507             :                         }
     508           0 :                         tm0 = GDKms();
     509           0 :                         fits_write_col(fptr, TFLOAT, cc+1, (optimal*block)+1, 1, dimension, readfloatrows, &status);
     510           0 :                         texportfloat += GDKms() - tm0;
     511           0 :                         GDKfree(readfloatrows);
     512             :                 }
     513             : 
     514           0 :                 if (strcmp(columntype,"double")==0)
     515             :                 {
     516           0 :                         doublecols++; dimension = 0; block = 0;
     517           0 :                         fits_get_rowsize(fptr,&optimal,&status);
     518           0 :                         readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal);
     519             : 
     520           0 :                         for (numberrow = 0; numberrow < nrows ; numberrow++)
     521             :                         {
     522           0 :                                 doublevalue = (double*) store->table_api.column_find_value(m->session->tr, col, (oid) numberrow);
     523           0 :                                 readdoublerows[dimension] = *doublevalue;
     524           0 :                                 GDKfree(doublevalue);
     525           0 :                                 dimension++;
     526             : 
     527           0 :                                 if (dimension == (size_t) optimal)
     528             :                                 {
     529             :                                         dimension = 0;
     530           0 :                                         tm0 = GDKms();
     531           0 :                                         fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status);
     532           0 :                                         texportdouble += GDKms() - tm0;
     533           0 :                                         GDKfree(readdoublerows);
     534           0 :                                         readdoublerows = (double *) GDKmalloc (sizeof(double) * optimal);
     535           0 :                                         block++;
     536             :                                 }
     537             :                         }
     538           0 :                         tm0 = GDKms();
     539           0 :                         fits_write_col(fptr, TDOUBLE, cc+1, (optimal*block)+1, 1, optimal, readdoublerows, &status);
     540           0 :                         texportdouble += GDKms() - tm0;
     541           0 :                         GDKfree(readdoublerows);
     542             :                 }
     543             :         }
     544             : 
     545             : 
     546             :         // print all the times that were needed to export each one of the columns
     547           0 :         if (texportboolean > 0)              TRC_DEBUG(FITS, "%d Boolean\tcolumn(s) exported in %d ms\n", boolcols, texportboolean);
     548           0 :         if (texportchar > 0)         TRC_DEBUG(FITS, "%d Char\t\tcolumn(s) exported in %d ms\n", charcols, texportchar);
     549           0 :         if (texportstring > 0)               TRC_DEBUG(FITS, "%d String\tcolumn(s) exported in %d ms\n", strcols, texportstring);
     550           0 :         if (texportshort > 0)                TRC_DEBUG(FITS, "%d Short\t\tcolumn(s) exported in %d ms\n", shortcols, texportshort);
     551           0 :         if (texportint > 0)                  TRC_DEBUG(FITS, "%d Integer\tcolumn(s) exported in %d ms\n", intcols, texportint);
     552           0 :         if (texportlng > 0)                  TRC_DEBUG(FITS, "%d Long\t\tcolumn(s) exported in %d ms\n", lngcols, texportlng);
     553           0 :         if (texportfloat > 0)                TRC_DEBUG(FITS, "%d Float\t\tcolumn(s) exported in %d ms\n", floatcols, texportfloat);
     554           0 :         if (texportdouble > 0)               TRC_DEBUG(FITS, "%d Double\tcolumn(s) exported in %d ms\n", doublecols, texportdouble);
     555             : 
     556             : 
     557           0 :         fits_close_file(fptr, &status);
     558           0 :         return msg;
     559             : }
     560             : 
     561             : 
     562           0 : str FITSdir(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
     563             : {
     564             :         str msg = MAL_SUCCEED;
     565           0 :         str dir = *getArgReference_str(stk, pci, 1);
     566             :         DIR *dp;
     567             :         struct dirent *ep;
     568             :         fitsfile *fptr;
     569             :         char *s;
     570           0 :         int status = 0;
     571             :         (void)mb;
     572             : 
     573           0 :         dp = opendir(dir);
     574           0 :         if (dp != NULL) {
     575             :                 char stmt[BUFSIZ];
     576             :                 char fname[BUFSIZ];
     577             : 
     578             :                 s = stmt;
     579             : 
     580           0 :                 while ((ep = readdir(dp)) != NULL && !msg) {
     581           0 :                         char *filename = SQLescapeString(ep->d_name);
     582           0 :                         if (!filename) {
     583           0 :                                 msg = createException(MAL, "fits.listdir", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     584           0 :                                 break;
     585             :                         }
     586             : 
     587           0 :                         snprintf(fname, BUFSIZ, "%s%s", dir, filename);
     588           0 :                         status = 0;
     589           0 :                         fits_open_file(&fptr, fname, READONLY, &status);
     590           0 :                         if (status == 0) {
     591           0 :                                 snprintf(stmt, BUFSIZ, ATTACHDIR, fname);
     592           0 :                                 TRC_DEBUG(FITS, "Executing: %s\n", s);
     593           0 :                                 msg = SQLstatementIntern(cntxt, s, "fits.listofdir", TRUE, FALSE, NULL);
     594           0 :                                 fits_close_file(fptr, &status);
     595             :                         }
     596             : 
     597           0 :                         GDKfree(filename);
     598             :                 }
     599           0 :                 (void)closedir(dp);
     600             :         } else
     601           0 :                 msg = createException(MAL, "listdir", SQLSTATE(FI000) "Couldn't open the directory");
     602             : 
     603           0 :         return msg;
     604             : }
     605             : 
     606           0 : str FITSdirpat(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
     607             : {
     608             :         str msg = MAL_SUCCEED;
     609           0 :         str dir = *getArgReference_str(stk, pci, 1);
     610           0 :         str pat = *getArgReference_str(stk, pci, 2);
     611             :         char *filename = NULL;
     612             :         fitsfile *fptr;
     613             :         char *s;
     614           0 :         int status = 0;
     615             :         glob_t globbuf;
     616             :         char fulldirectory[BUFSIZ];
     617             :         size_t j = 0;
     618             : 
     619             :         (void)mb;
     620             : 
     621           0 :         globbuf.gl_offs = 0;
     622           0 :         snprintf(fulldirectory, BUFSIZ, "%s%s", dir, pat);
     623           0 :         glob(fulldirectory, GLOB_DOOFFS, NULL, &globbuf);
     624             : 
     625           0 :         TRC_DEBUG(FITS, "Fulldir: %s - Size: %zu\n", fulldirectory, globbuf.gl_pathc);
     626             : 
     627           0 :         if (globbuf.gl_pathc == 0)
     628           0 :                 throw(MAL, "fits.listdirpat", SQLSTATE(FI000) "Couldn't open the directory or there are no files that match the pattern");
     629             : 
     630           0 :         for (j = 0; j < globbuf.gl_pathc; j++) {
     631             :                 char stmt[BUFSIZ];
     632             :                 char fname[BUFSIZ];
     633             : 
     634             :                 s = stmt;
     635           0 :                 snprintf(fname, BUFSIZ, "%s", globbuf.gl_pathv[j]);
     636           0 :                 filename = SQLescapeString(fname);
     637           0 :                 if (!filename) {
     638           0 :                         throw(MAL, "fits.listdirpat", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     639             :                 }
     640           0 :                 status = 0;
     641           0 :                 fits_open_file(&fptr, filename, READONLY, &status);
     642           0 :                 if (status == 0) {
     643           0 :                         snprintf(stmt, BUFSIZ, ATTACHDIR, filename);
     644           0 :                         GDKfree(filename);
     645           0 :                         TRC_DEBUG(FITS, "Executing: %s\n", s);
     646           0 :                         msg = SQLstatementIntern(cntxt, s, "fits.listofdirpat", TRUE, FALSE, NULL);
     647           0 :                         fits_close_file(fptr, &status);
     648             : 
     649           0 :                         break;
     650             :                 }
     651             :         }
     652             : 
     653             :         return msg;
     654             : }
     655             : 
     656             : 
     657             : str
     658           0 : FITStest(int *res, str *fname)
     659             : {
     660             :         fitsfile *fptr;       /* pointer to the FITS file, defined in fitsio.h */
     661             :         str msg = MAL_SUCCEED;
     662           0 :         int status = 0, hdutype;
     663             : 
     664           0 :         *res = 0;
     665           0 :         if (fits_open_file(&fptr, *fname, READONLY, &status))
     666           0 :                 msg = createException(MAL, "fits.test", SQLSTATE(FI000) "Missing FITS file %s", *fname);
     667             :         else {
     668           0 :                 fits_movabs_hdu(fptr, 2, &hdutype, &status);
     669           0 :                 *res = hdutype;
     670           0 :                 fits_close_file(fptr, &status);
     671             :         }
     672             : 
     673           0 :         return msg;
     674             : }
     675             : 
     676           0 : str FITSattach(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
     677             : {
     678           0 :         mvc *m = NULL;
     679             :         sql_trans *tr;
     680             :         sql_schema *sch;
     681             :         sql_table *fits_tp, *fits_fl, *fits_tbl, *fits_col, *tbl = NULL;
     682             :         sql_column *col;
     683             :         str msg = MAL_SUCCEED;
     684           0 :         str fname = *getArgReference_str(stk, pci, 1);
     685             :         fitsfile *fptr;  /* pointer to the FITS file */
     686           0 :         int status = 0, i, j, hdutype, hdunum = 1, cnum = 0, bitpixnumber = 0;
     687             :         oid fid, tid, cid, rid = oid_nil;
     688           0 :         char tname[BUFSIZ], *tname_low = NULL, *s, bname[BUFSIZ], stmt[BUFSIZ];
     689             :         long tbcol; /* type long used by fits library */
     690             :         char cname[BUFSIZ], tform[BUFSIZ], tunit[BUFSIZ], tnull[BUFSIZ], tdisp[BUFSIZ];
     691             :         char *esc_cname, *esc_tform, *esc_tunit;
     692             :         double tscal, tzero;
     693           0 :         char xtensionname[BUFSIZ] = "", stilversion[BUFSIZ] = "";
     694           0 :         char stilclass[BUFSIZ] = "", tdate[BUFSIZ] = "", orig[BUFSIZ] = "", comm[BUFSIZ] = "";
     695             : 
     696           0 :         if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
     697             :                 return msg;
     698           0 :         if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
     699             :                 return msg;
     700             : 
     701           0 :         if (fits_open_file(&fptr, fname, READONLY, &status)) {
     702           0 :                 msg = createException(MAL, "fits.attach", SQLSTATE(FI000) "Missing FITS file %s.\n", fname);
     703           0 :                 return msg;
     704             :         }
     705             : 
     706           0 :         tr = m->session->tr;
     707           0 :         sch = mvc_bind_schema(m, "sys");
     708           0 :         sqlstore *store = tr->store;
     709             : 
     710           0 :         fits_fl = mvc_bind_table(m, sch, "fits_files");
     711           0 :         if (fits_fl == NULL)
     712           0 :                 FITSinitCatalog(m);
     713             : 
     714           0 :         fits_fl = mvc_bind_table(m, sch, "fits_files");
     715           0 :         fits_tbl = mvc_bind_table(m, sch, "fits_tables");
     716           0 :         fits_col = mvc_bind_table(m, sch, "fits_columns");
     717           0 :         fits_tp = mvc_bind_table(m, sch, "fits_table_properties");
     718             : 
     719             :         /* check if the file is already attached */
     720           0 :         col = mvc_bind_column(m, fits_fl, "name");
     721           0 :         rid = store->table_api.column_find_row(m->session->tr, col, fname, NULL);
     722           0 :         if (!is_oid_nil(rid)) {
     723           0 :                 fits_close_file(fptr, &status);
     724           0 :                 throw(MAL, "fits.attach", SQLSTATE(FI000) "File %s already attached\n", fname);
     725             :         }
     726             : 
     727             :         /* add row in the fits_files catalog table */
     728             :         BUN offset;
     729           0 :         if (store->storage_api.claim_tab(m->session->tr, fits_fl, 1, &offset, NULL) != LOG_OK) {
     730           0 :                 fits_close_file(fptr, &status);
     731           0 :                 throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     732             :         }
     733           0 :         if (!isNew(fits_fl) && sql_trans_add_dependency_change(m->session->tr, fits_fl->base.id, dml) != LOG_OK) {
     734           0 :                 fits_close_file(fptr, &status);
     735           0 :                 throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     736             :         }
     737           0 :         col = mvc_bind_column(m, fits_fl, "id");
     738           0 :         fid = store->storage_api.count_col(tr, col, 0) + 1;
     739           0 :         store->storage_api.append_col(m->session->tr,
     740             :                 mvc_bind_column(m, fits_fl, "id"), offset, NULL, &fid, 1, TYPE_int);
     741           0 :         store->storage_api.append_col(m->session->tr,
     742             :                 mvc_bind_column(m, fits_fl, "name"), offset, NULL, &fname, 1, TYPE_str);
     743           0 :         col = mvc_bind_column(m, fits_tbl, "id");
     744           0 :         tid = store->storage_api.count_col(tr, col, 0) + 1;
     745             : 
     746           0 :         if ((s = strrchr(fname, DIR_SEP)) == NULL)
     747             :                 s = fname;
     748             :         else
     749           0 :                 s++;
     750           0 :         strcpy(bname, s);
     751           0 :         s = strrchr(bname, '.');
     752           0 :         if (s) *s = 0;
     753             : 
     754           0 :         fits_get_num_hdus(fptr, &hdunum, &status);
     755           0 :         for (i = 1; i <= hdunum; i++) {
     756           0 :                 fits_movabs_hdu(fptr, i, &hdutype, &status);
     757           0 :                 if (hdutype != ASCII_TBL && hdutype != BINARY_TBL)
     758           0 :                         continue;
     759             : 
     760             :                 /* SQL table name - the name of FITS extention */
     761           0 :                 fits_read_key(fptr, TSTRING, "EXTNAME", tname, NULL, &status);
     762           0 :                 if (status) {
     763           0 :                         snprintf(tname, BUFSIZ, "%s_%d", bname, i);
     764           0 :                         tname_low = toLower(tname);
     765           0 :                         status = 0;
     766             :                 }else  { /* check table name for existence in the fits catalog */
     767           0 :                         tname_low = toLower(tname);
     768           0 :                         col = mvc_bind_column(m, fits_tbl, "name");
     769           0 :                         rid = store->table_api.column_find_row(m->session->tr, col, tname_low, NULL);
     770             :                         /* or as regular SQL table */
     771           0 :                         tbl = mvc_bind_table(m, sch, tname_low);
     772           0 :                         if (!is_oid_nil(rid) || tbl) {
     773           0 :                                 snprintf(tname, BUFSIZ, "%s_%d", bname, i);
     774           0 :                                 tname_low = toLower(tname);
     775             :                         }
     776             :                 }
     777             : 
     778           0 :                 fits_read_key(fptr, TSTRING, "BITPIX", &bitpixnumber, NULL, &status);
     779           0 :                 if (status) {
     780           0 :                         status = 0;
     781             :                 }
     782           0 :                 fits_read_key(fptr, TSTRING, "DATE-HDU", tdate, NULL, &status);
     783           0 :                 if (status) {
     784           0 :                         status = 0;
     785             :                 }
     786           0 :                 fits_read_key(fptr, TSTRING, "XTENSION", xtensionname, NULL, &status);
     787           0 :                 if (status) {
     788           0 :                         status = 0;
     789             :                 }
     790           0 :                 fits_read_key(fptr, TSTRING, "STILVERS", stilversion, NULL, &status);
     791           0 :                 if (status) {
     792           0 :                         status = 0;
     793             :                 }
     794           0 :                 fits_read_key(fptr, TSTRING, "STILCLAS", stilclass, NULL, &status);
     795           0 :                 if (status) {
     796           0 :                         status = 0;
     797             :                 }
     798           0 :                 fits_read_key(fptr, TSTRING, "ORIGIN", orig, NULL, &status);
     799           0 :                 if (status) {
     800           0 :                         status = 0;
     801             :                 }
     802           0 :                 fits_read_key(fptr, TSTRING, "COMMENT", comm, NULL, &status);
     803           0 :                 if (status) {
     804           0 :                         status = 0;
     805             :                 }
     806             : 
     807           0 :                 fits_get_num_cols(fptr, &cnum, &status);
     808             : 
     809             :                 BUN offset;
     810           0 :                 if (store->storage_api.claim_tab(m->session->tr, fits_tbl, 1, &offset, NULL) != LOG_OK) {
     811           0 :                         fits_close_file(fptr, &status);
     812           0 :                         throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     813             :                 }
     814           0 :                 if (!isNew(fits_tbl) && sql_trans_add_dependency_change(m->session->tr, fits_tbl->base.id, dml) != LOG_OK) {
     815           0 :                         fits_close_file(fptr, &status);
     816           0 :                         throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     817             :                 }
     818           0 :                 store->storage_api.append_col(m->session->tr,
     819             :                         mvc_bind_column(m, fits_tbl, "id"), offset, NULL, &tid, 1, TYPE_int);
     820           0 :                 store->storage_api.append_col(m->session->tr,
     821             :                         mvc_bind_column(m, fits_tbl, "name"), offset, NULL, &tname_low, 1, TYPE_str);
     822           0 :                 store->storage_api.append_col(m->session->tr,
     823             :                         mvc_bind_column(m, fits_tbl, "columns"), offset, NULL, &cnum, 1, TYPE_int);
     824           0 :                 store->storage_api.append_col(m->session->tr,
     825             :                         mvc_bind_column(m, fits_tbl, "file_id"), offset, NULL, &fid, 1, TYPE_int);
     826           0 :                 store->storage_api.append_col(m->session->tr,
     827             :                         mvc_bind_column(m, fits_tbl, "hdu"), offset, NULL, &i, 1, TYPE_int);
     828           0 :                 char *vptr = tdate;
     829           0 :                 store->storage_api.append_col(m->session->tr,
     830             :                         mvc_bind_column(m, fits_tbl, "date"), offset, NULL, &vptr, 1, TYPE_str);
     831           0 :                 vptr = orig;
     832           0 :                 store->storage_api.append_col(m->session->tr,
     833             :                         mvc_bind_column(m, fits_tbl, "origin"), offset, NULL, &vptr, 1, TYPE_str);
     834           0 :                 vptr = comm;
     835           0 :                 store->storage_api.append_col(m->session->tr,
     836             :                         mvc_bind_column(m, fits_tbl, "comment"), offset, NULL, &vptr, 1, TYPE_str);
     837             : 
     838           0 :                 if (store->storage_api.claim_tab(m->session->tr, fits_tp, 1, &offset, NULL) != LOG_OK) {
     839           0 :                         fits_close_file(fptr, &status);
     840           0 :                         throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     841             :                 }
     842           0 :                 if (!isNew(fits_tp) && sql_trans_add_dependency_change(m->session->tr, fits_tp->base.id, dml) != LOG_OK) {
     843           0 :                         fits_close_file(fptr, &status);
     844           0 :                         throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     845             :                 }
     846           0 :                 store->storage_api.append_col(m->session->tr,
     847             :                         mvc_bind_column(m, fits_tp, "table_id"), offset, NULL, &tid, 1, TYPE_int);
     848           0 :                 vptr = xtensionname;
     849           0 :                 store->storage_api.append_col(m->session->tr,
     850             :                         mvc_bind_column(m, fits_tp, "xtension"), offset, NULL, &vptr, 1, TYPE_str);
     851           0 :                 store->storage_api.append_col(m->session->tr,
     852             :                         mvc_bind_column(m, fits_tp, "bitpix"), offset, NULL, &bitpixnumber, 1, TYPE_int);
     853           0 :                 vptr = stilversion;
     854           0 :                 store->storage_api.append_col(m->session->tr,
     855             :                         mvc_bind_column(m, fits_tp, "stilvers"), offset, NULL, &vptr, 1, TYPE_str);
     856           0 :                 vptr = stilclass;
     857           0 :                 store->storage_api.append_col(m->session->tr,
     858             :                         mvc_bind_column(m, fits_tp, "stilclas"), offset, NULL, &vptr, 1, TYPE_str);
     859             : 
     860             :                 /* read columns description */
     861             :                 s = stmt;
     862           0 :                 col = mvc_bind_column(m, fits_col, "id");
     863           0 :                 cid = store->storage_api.count_col(tr, col, 0) + 1;
     864           0 :                 for (j = 1; j <= cnum; j++, cid++) {
     865           0 :                         fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status);
     866             :                         /* escape the various strings to avoid SQL injection attacks */
     867           0 :                         esc_cname = SQLescapeString(cname);
     868           0 :                         if (!esc_cname) {
     869           0 :                                 fits_close_file(fptr, &status);
     870           0 :                                 throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     871             :                         }
     872           0 :                         esc_tform = SQLescapeString(tform);
     873           0 :                         if (!esc_tform) {
     874           0 :                                 GDKfree(esc_cname);
     875           0 :                                 fits_close_file(fptr, &status);
     876           0 :                                 throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     877             :                         }
     878           0 :                         esc_tunit = SQLescapeString(tunit);
     879             :                         if (!esc_tform) {
     880             :                                 GDKfree(esc_tform);
     881             :                                 GDKfree(esc_cname);
     882             :                                 fits_close_file(fptr, &status);
     883             :                                 throw(MAL, "fits.attach", SQLSTATE(HY013) MAL_MALLOC_FAIL);
     884             :                         }
     885           0 :                         snprintf(stmt, BUFSIZ, FITS_INS_COL, (int)cid, esc_cname, esc_tform, esc_tunit, j, (int)tid);
     886           0 :                         GDKfree(esc_tunit);
     887           0 :                         GDKfree(esc_tform);
     888           0 :                         GDKfree(esc_cname);
     889           0 :                         msg = SQLstatementIntern(cntxt, s, "fits.attach", TRUE, FALSE, NULL);
     890           0 :                         if (msg != MAL_SUCCEED) {
     891           0 :                                 fits_close_file(fptr, &status);
     892           0 :                                 return msg;
     893             :                         }
     894             :                 }
     895           0 :                 tid++;
     896             :         }
     897           0 :         fits_close_file(fptr, &status);
     898             : 
     899           0 :         return MAL_SUCCEED;
     900             : }
     901             : 
     902           0 : str FITSloadTable(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
     903             : {
     904           0 :         mvc *m = NULL;
     905             :         sql_schema *sch;
     906           0 :         sql_table *fits_fl, *fits_tbl, *tbl = NULL;
     907             :         sql_column *col;
     908             :         sql_subtype tpe;
     909             :         fitsfile *fptr;
     910           0 :         str tname = *getArgReference_str(stk, pci, 1);
     911             :         str fname;
     912             :         str msg = MAL_SUCCEED;
     913             :         oid rid = oid_nil, frid = oid_nil;
     914           0 :         int status = 0, cnum = 0, *fid, *hdu, hdutype, j, anynull = 0, mtype;
     915             :         int *tpcode = NULL;
     916             :         long *rep = NULL, *wid = NULL, rows; /* type long used by fits library */
     917             :         char keywrd[80], **cname, nm[FLEN_VALUE];
     918             :         const void *nilptr;
     919             :         BUN offset;
     920           0 :         BAT *pos = NULL;
     921             : 
     922           0 :         if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != MAL_SUCCEED)
     923             :                 return msg;
     924           0 :         if ((msg = checkSQLContext(cntxt)) != MAL_SUCCEED)
     925             :                 return msg;
     926           0 :         sch = mvc_bind_schema(m, "sys");
     927             : 
     928           0 :         fits_tbl = mvc_bind_table(m, sch, "fits_tables");
     929           0 :         if (fits_tbl == NULL) {
     930           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "FITS catalog is missing.\n");
     931           0 :                 return msg;
     932             :         }
     933             : 
     934           0 :         tbl = mvc_bind_table(m, sch, tname);
     935           0 :         if (tbl) {
     936           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Table %s is already created.\n", tname);
     937           0 :                 return msg;
     938             :         }
     939             : 
     940           0 :         col = mvc_bind_column(m, fits_tbl, "name");
     941           0 :         sqlstore *store = m->session->tr->store;
     942           0 :         rid = store->table_api.column_find_row(m->session->tr, col, tname, NULL);
     943           0 :         if (is_oid_nil(rid)) {
     944           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Table %s is unknown in FITS catalog. Attach first the containing file\n", tname);
     945           0 :                 return msg;
     946             :         }
     947             : 
     948             :         /* Open FITS file and move to the table HDU */
     949           0 :         col = mvc_bind_column(m, fits_tbl, "file_id");
     950           0 :         fid = (int*)store->table_api.column_find_value(m->session->tr, col, rid);
     951             : 
     952           0 :         fits_fl = mvc_bind_table(m, sch, "fits_files");
     953           0 :         col = mvc_bind_column(m, fits_fl, "id");
     954           0 :         frid = store->table_api.column_find_row(m->session->tr, col, (void *)fid, NULL);
     955           0 :         GDKfree(fid);
     956           0 :         if (is_oid_nil(frid)) {
     957           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Table %s is unknown in FITS catalog. Attach first the containing file\n", tname);
     958           0 :                 return msg;
     959             :         }
     960           0 :         col = mvc_bind_column(m, fits_fl, "name");
     961           0 :         fname = (char *)store->table_api.column_find_value(m->session->tr, col, frid);
     962           0 :         if (fits_open_file(&fptr, fname, READONLY, &status)) {
     963           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Missing FITS file %s.\n", fname);
     964           0 :                 GDKfree(fname);
     965           0 :                 return msg;
     966             :         }
     967           0 :         GDKfree(fname);
     968             : 
     969           0 :         col = mvc_bind_column(m, fits_tbl, "hdu");
     970           0 :         hdu = (int*)store->table_api.column_find_value(m->session->tr, col, rid);
     971           0 :         fits_movabs_hdu(fptr, *hdu, &hdutype, &status);
     972           0 :         if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) {
     973           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "HDU %d is not a table.\n", *hdu);
     974           0 :                 GDKfree(hdu);
     975           0 :                 fits_close_file(fptr, &status);
     976           0 :                 return msg;
     977             :         }
     978           0 :         GDKfree(hdu);
     979             : 
     980             :         /* create a SQL table to hold the FITS table */
     981             :         /*      col = mvc_bind_column(m, fits_tbl, "columns");
     982             :            cnum = *(int*) store->table_api.column_find_value(m->session->tr, col, rid); */
     983           0 :         fits_get_num_cols(fptr, &cnum, &status);
     984           0 :         mvc_create_table(&tbl, m, sch, tname, tt_table, 0, SQL_PERSIST, 0, cnum, 0);
     985             : 
     986             :         // TODO: Check that the allocations succeeded
     987           0 :         tpcode = (int *)GDKzalloc(sizeof(int) * cnum);
     988           0 :         rep = (long *)GDKzalloc(sizeof(long) * cnum);
     989           0 :         wid = (long *)GDKzalloc(sizeof(long) * cnum);
     990           0 :         cname = (char **)GDKzalloc(sizeof(char *) * cnum);
     991             : 
     992           0 :         for (j = 1; j <= cnum; j++) {
     993           0 :                 sql_column *col = NULL;
     994             :                 /*              fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform, &tscal, &tzero, tnull, tdisp, &status); */
     995           0 :                 snprintf(keywrd, 80, "TTYPE%d", j);
     996           0 :                 fits_read_key(fptr, TSTRING, keywrd, nm, NULL, &status);
     997           0 :                 if (status) {
     998           0 :                         snprintf(nm, FLEN_VALUE, "column_%d", j);
     999           0 :                         status = 0;
    1000             :                 }
    1001           0 :                 cname[j - 1] = toLower(nm);
    1002           0 :                 fits_get_coltype(fptr, j, &tpcode[j - 1], &rep[j - 1], &wid[j - 1], &status);
    1003           0 :                 fits2subtype(&tpe, tpcode[j - 1], rep[j - 1], wid[j - 1]);
    1004             : 
    1005           0 :                 TRC_DEBUG(FITS, "%d %ld %ld - M: %s\n", tpcode[j-1], rep[j-1], wid[j-1], tpe.type->base.name);
    1006             : 
    1007           0 :                 mvc_create_column(&col, m, tbl, cname[j - 1], &tpe);
    1008             :         }
    1009             : 
    1010             :         /* data load */
    1011           0 :         fits_get_num_rows(fptr, &rows, &status);
    1012             :         /* Nothing more to do */
    1013           0 :         if (rows == 0) {
    1014           0 :                 goto bailout;
    1015             :         }
    1016             : 
    1017           0 :         TRC_INFO(FITS, "Loading %ld rows in table %s\n", rows, tname);
    1018             : 
    1019           0 :         if (store->storage_api.claim_tab(m->session->tr, tbl, rows, &offset, &pos) != LOG_OK) {
    1020           0 :                 msg = createException(MAL, "fits.loadtable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1021           0 :                 goto bailout;
    1022             :         }
    1023           0 :         for (j = 1; j <= cnum; j++) {
    1024             :                 BAT *tmp = NULL;
    1025           0 :                 int time0 = GDKms();
    1026           0 :                 mtype = fits2mtype(tpcode[j - 1], rep[j - 1]);
    1027           0 :                 nilptr = ATOMnilptr(mtype);
    1028           0 :                 col = mvc_bind_column(m, tbl, cname[j - 1]);
    1029             : 
    1030           0 :                 tmp = COLnew(0, mtype, rows, TRANSIENT);
    1031           0 :                 if (tmp == NULL){
    1032           0 :                         msg = createException(MAL,"fits.load", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1033           0 :                         goto bailout;
    1034             :                 }
    1035           0 :                 if (mtype == TYPE_blob) {
    1036             :                         long i;
    1037           0 :                         unsigned long nbytes = rep[j - 1] * wid[j - 1];
    1038           0 :                         blob **v = (blob **)GDKzalloc(sizeof(blob *) * rows);
    1039             : 
    1040           0 :                         mtype = fits2mtype(tpcode[j - 1], 1);
    1041           0 :                         nilptr = ATOMnilptr(mtype);
    1042             : 
    1043           0 :                         if (v == NULL) {
    1044           0 :                                 BBPreclaim(tmp);
    1045           0 :                                 msg = createException(MAL,"fits.load", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1046           0 :                                 goto bailout;
    1047             :                         }
    1048             : 
    1049           0 :                         for(i = 0; i < rows; i++) {
    1050           0 :                                 v[i] = (blob *)GDKmalloc(offsetof(blob, data) + nbytes);
    1051           0 :                                 if (v[i] == NULL) {
    1052           0 :                                         BBPreclaim(tmp);
    1053             :                                         long k = 0;
    1054           0 :                                         for (k = 0; k < i; k++) {
    1055           0 :                                                 GDKfree(v[k]);
    1056             :                                         }
    1057           0 :                                         GDKfree(v);
    1058           0 :                                         msg = createException(MAL,"fits.load", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1059           0 :                                         goto bailout;
    1060             :                                 }
    1061           0 :                                 fits_read_col(fptr, tpcode[j - 1], j, i + 1, 1, rep[j - 1], (void *)nilptr,
    1062           0 :                                               (void *)v[i]->data, &anynull, &status);
    1063           0 :                                 v[i]->nitems = nbytes;
    1064           0 :                                 if (BUNappend(tmp, v[i], false) != GDK_SUCCEED) {
    1065           0 :                                         BBPreclaim(tmp);
    1066           0 :                                         for (i = 0; i < rows; i++) {
    1067           0 :                                                 GDKfree(v[i]);
    1068             :                                         }
    1069           0 :                                         GDKfree(v);
    1070           0 :                                         msg = createException(MAL,"fits.loadtable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1071           0 :                                         goto bailout;
    1072             :                                 }
    1073             :                         }
    1074             : 
    1075           0 :                         for(i = 0; i < rows; i++) {
    1076           0 :                                 GDKfree(v[i]);
    1077             :                         }
    1078           0 :                         GDKfree(v);
    1079             :                 }
    1080           0 :                 else if (mtype == TYPE_str) {
    1081             : /*                      char *v = GDKzalloc(wid[j-1]);*/
    1082             :                         /* type long demanded by "rows", i.e., by fits library */
    1083             :                         long bsize = 50, batch = bsize, k, i;
    1084             :                         int tm0, tloadtm = 0, tattachtm = 0;
    1085           0 :                         char **v = (char **) GDKzalloc(sizeof(char *) * bsize);
    1086           0 :                         for(i = 0; i < bsize; i++)
    1087           0 :                                 v[i] = GDKzalloc(wid[j-1]);
    1088           0 :                         for(i = 0; i < rows; i += batch) {
    1089           0 :                                 batch = rows - i < bsize ? rows - i: bsize;
    1090           0 :                                 tm0 = GDKms();
    1091           0 :                                 fits_read_col(fptr, tpcode[j - 1], j, 1 + i, 1, batch, (void *) nilptr, (void *)v, &anynull, &status);
    1092           0 :                                 tloadtm += GDKms() - tm0;
    1093           0 :                                 tm0 = GDKms();
    1094           0 :                                 for(k = 0; k < batch ; k++)
    1095           0 :                                         if (BUNappend(tmp, v[k], false) != GDK_SUCCEED) {
    1096           0 :                                                 BBPreclaim(tmp);
    1097           0 :                                                 msg = createException(MAL, "fits.loadtable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1098           0 :                                                 goto bailout;
    1099             :                                         }
    1100           0 :                                 tattachtm += GDKms() - tm0;
    1101             :                         }
    1102           0 :                         for(i = 0; i < bsize ; i++)
    1103           0 :                                 GDKfree(v[i]);
    1104           0 :                         GDKfree(v);
    1105           0 :                         TRC_INFO(FITS, "String column load %d ms, BUNappend %d ms\n", tloadtm, tattachtm);
    1106             :                 }
    1107             :                 else {
    1108           0 :                         BATiter bi = bat_iterator_nolock(tmp);
    1109           0 :                         fits_read_col(fptr, tpcode[j - 1], j, 1, 1, rows, (void *) nilptr, (void *)BUNtloc(bi, 0), &anynull, &status);
    1110           0 :                         BATsetcount(tmp, rows);
    1111           0 :                         tmp->tsorted = false;
    1112           0 :                         tmp->trevsorted = false;
    1113             :                 }
    1114             : 
    1115           0 :                 if (status) {
    1116             :                         char buf[FLEN_ERRMSG + 1];
    1117           0 :                         fits_read_errmsg(buf);
    1118           0 :                         BBPreclaim(tmp);
    1119           0 :                         msg = createException(MAL, "fits.loadtable", SQLSTATE(FI000) "Cannot load column %s of %s table: %s.\n", cname[j - 1], tname, buf);
    1120             :                         break;
    1121             :                 }
    1122             : 
    1123           0 :                 TRC_INFO(FITS, "#Column %s loaded for %d ms\t", cname[j-1], GDKms() - time0);
    1124           0 :                 if (store->storage_api.append_col(m->session->tr, col, offset, pos, tmp, BATcount(tmp), TYPE_bat) != LOG_OK) {
    1125           0 :                         BBPreclaim(tmp);
    1126           0 :                         msg = createException(MAL, "fits.loadtable", SQLSTATE(HY013) MAL_MALLOC_FAIL);
    1127           0 :                         break;
    1128             :                 }
    1129           0 :                 TRC_INFO(FITS, "Total %d ms\n", GDKms() - time0);
    1130           0 :                 BBPreclaim(tmp);
    1131             :         }
    1132             : 
    1133           0 :   bailout:
    1134           0 :         bat_destroy(pos);
    1135           0 :         GDKfree(tpcode);
    1136           0 :         GDKfree(rep);
    1137           0 :         GDKfree(wid);
    1138           0 :         GDKfree(cname);
    1139             : 
    1140           0 :         fits_close_file(fptr, &status);
    1141           0 :         return msg;
    1142             : }
    1143             : 
    1144             : #include "mel.h"
    1145             : static mel_func fits_init_funcs[] = {
    1146             :  pattern("fits", "listdir", FITSdir, false, "Attach all FITS files in the directory", args(1,2, arg("",void),arg("dirname",str))),
    1147             :  pattern("fits", "listdirpattern", FITSdirpat, false, "Attach all FITS file in the directory, giving a pattern", args(1,3, arg("",void),arg("dirname",str),arg("pattern",str))),
    1148             :  command("fits", "fitstest", FITStest, false, "Returns the type of first extension in the FITS file filename", args(1,2, arg("",int),arg("filename",str))),
    1149             :  pattern("fits", "attach", FITSattach, false, "Open a FITS file and return catalog of the table HDUs", args(1,2, arg("",void),arg("fname",str))),
    1150             :  pattern("fits", "load", FITSloadTable, false, "Load a FITS table from an attached file", args(1,2, arg("",void),arg("tablename",str))),
    1151             :  pattern("fits", "export", FITSexportTable, false, "Export a table to a FITS file", args(1,2, arg("",void),arg("tablename",str))),
    1152             :  { .imp=NULL }
    1153             : };
    1154             : #include "mal_import.h"
    1155             : #ifdef _MSC_VER
    1156             : #undef read
    1157             : #pragma section(".CRT$XCU",read)
    1158             : #endif
    1159         254 : LIB_STARTUP_FUNC(init_fits_mal)
    1160         254 : { mal_module("fits", NULL, fits_init_funcs); }

Generated by: LCOV version 1.14