Line data Source code
1 : /*
2 : * This Source Code Form is subject to the terms of the Mozilla Public
3 : * License, v. 2.0. If a copy of the MPL was not distributed with this
4 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 : *
6 : * Copyright 1997 - July 2008 CWI, August 2008 - 2021 MonetDB B.V.
7 : */
8 :
9 : /* streams working on a gzip-compressed disk file */
10 :
11 : #include "monetdb_config.h"
12 : #include "stream.h"
13 : #include "stream_internal.h"
14 :
15 :
16 :
17 : /* ------------------------------------------------------------------ */
18 : /* streams working on a remote file using cURL */
19 :
20 : #ifdef HAVE_CURL
21 : #include <curl/curl.h>
22 :
23 : struct curl_data {
24 : CURL *handle;
25 : char *buffer; /* buffer to store incoming data */
26 : size_t maxsize; /* size of allocated buffer */
27 : size_t usesize; /* end of used data */
28 : size_t offset; /* start of unread data */
29 : int running; /* whether still transferring */
30 : char errbuf[CURL_ERROR_SIZE]; /* a place for error messages */
31 : };
32 :
33 : #define BLOCK_CURL ((size_t) 1 << 16)
34 :
35 : /* this function is called by libcurl when there is data for us */
36 : static size_t
37 0 : write_callback(char *buffer, size_t size, size_t nitems, void *userp)
38 : {
39 : stream *s = (stream *) userp;
40 0 : struct curl_data *c = (struct curl_data *) s->stream_data.p;
41 :
42 0 : size *= nitems;
43 0 : if (size == 0) /* unlikely */
44 : return 0;
45 : /* allocate a buffer if we don't have one yet */
46 0 : if (c->buffer == NULL) {
47 : /* BLOCK_CURL had better be a power of 2! */
48 0 : c->maxsize = (size + BLOCK_CURL - 1) & ~(BLOCK_CURL - 1);
49 0 : if ((c->buffer = malloc(c->maxsize)) == NULL)
50 : return 0;
51 0 : c->usesize = 0;
52 0 : c->offset = 0;
53 : }
54 : /* move data if we don't have enough space */
55 0 : if (c->maxsize - c->usesize < size && c->offset > 0) {
56 0 : memmove(c->buffer, c->buffer + c->offset, c->usesize - c->offset);
57 0 : c->usesize -= c->offset;
58 0 : c->offset = 0;
59 : }
60 : /* allocate more buffer space if we still don't have enough space */
61 0 : if (c->maxsize - c->usesize < size) {
62 : char *b;
63 : size_t maxsize;
64 :
65 0 : maxsize = (c->usesize + size + BLOCK_CURL - 1) & ~(BLOCK_CURL - 1);
66 0 : b = realloc(c->buffer, maxsize);
67 0 : if (b == NULL)
68 : return 0; /* indicate failure to library */
69 0 : c->buffer = b;
70 0 : c->maxsize = maxsize;
71 : }
72 : /* finally, store the data we received */
73 0 : memcpy(c->buffer + c->usesize, buffer, size);
74 0 : c->usesize += size;
75 0 : return size;
76 : }
77 :
78 : static void
79 0 : curl_destroy(stream *s)
80 : {
81 : struct curl_data *c;
82 :
83 0 : if ((c = (struct curl_data *) s->stream_data.p) != NULL) {
84 0 : s->stream_data.p = NULL;
85 0 : if (c->handle) {
86 0 : curl_easy_cleanup(c->handle);
87 : }
88 0 : if (c->buffer)
89 0 : free(c->buffer);
90 0 : free(c);
91 : }
92 0 : destroy_stream(s);
93 0 : }
94 :
95 : static ssize_t
96 0 : curl_read(stream *restrict s, void *restrict buf, size_t elmsize, size_t cnt)
97 : {
98 0 : struct curl_data *c = (struct curl_data *) s->stream_data.p;
99 0 : size_t size = cnt * elmsize;
100 :
101 0 : if (c == NULL) {
102 0 : mnstr_set_error(s, MNSTR_READ_ERROR, "stream already ended");
103 0 : return -1;
104 : }
105 :
106 0 : if (size == 0)
107 : return 0;
108 0 : if (c->usesize - c->offset >= elmsize || !c->running) {
109 : /* there is at least one element's worth of data
110 : * available, or we have reached the end: return as
111 : * much as we have, but no more than requested */
112 0 : if (size > c->usesize - c->offset) {
113 0 : cnt = (c->usesize - c->offset) / elmsize;
114 0 : size = cnt * elmsize;
115 : }
116 0 : memcpy(buf, c->buffer + c->offset, size);
117 0 : c->offset += size;
118 0 : if (c->offset == c->usesize)
119 0 : c->usesize = c->offset = 0;
120 0 : return (ssize_t) cnt;
121 : }
122 : /* not enough data, we must wait until we get some */
123 : return 0;
124 : }
125 :
126 : static ssize_t
127 0 : curl_write(stream *restrict s, const void *restrict buf, size_t elmsize, size_t cnt)
128 : {
129 : (void) s;
130 : (void) buf;
131 : (void) elmsize;
132 : (void) cnt;
133 0 : assert(0);
134 : return -1;
135 : }
136 :
137 : static void
138 0 : curl_close(stream *s)
139 : {
140 : (void) s;
141 0 : }
142 :
143 : stream *
144 0 : open_urlstream(const char *url)
145 : {
146 : stream *s;
147 : struct curl_data *c;
148 :
149 0 : if ((c = malloc(sizeof(*c))) == NULL) {
150 0 : mnstr_set_open_error(url, errno, NULL);
151 0 : return NULL;
152 : }
153 0 : *c = (struct curl_data) {
154 : .running = 1,
155 : .errbuf = {0},
156 : };
157 0 : if ((s = create_stream(url)) == NULL) {
158 0 : free(c);
159 0 : return NULL;
160 : }
161 0 : s->read = curl_read;
162 0 : s->write = curl_write;
163 0 : s->close = curl_close;
164 0 : s->destroy = curl_destroy;
165 0 : if ((c->handle = curl_easy_init()) == NULL) {
166 0 : free(c);
167 0 : destroy_stream(s);
168 0 : mnstr_set_open_error(url, 0, "curl_easy_init failed");
169 0 : return NULL;
170 : }
171 0 : s->stream_data.p = (void *) c;
172 0 : curl_easy_setopt(c->handle, CURLOPT_URL, s->name);
173 0 : curl_easy_setopt(c->handle, CURLOPT_WRITEDATA, s);
174 0 : curl_easy_setopt(c->handle, CURLOPT_VERBOSE, 0);
175 0 : curl_easy_setopt(c->handle, CURLOPT_NOSIGNAL, 1);
176 0 : curl_easy_setopt(c->handle, CURLOPT_FAILONERROR, 1);
177 0 : curl_easy_setopt(c->handle, CURLOPT_ERRORBUFFER, c->errbuf);
178 0 : curl_easy_setopt(c->handle, CURLOPT_WRITEFUNCTION, write_callback);
179 0 : CURLcode ret = curl_easy_perform(c->handle);
180 0 : if (ret != CURLE_OK) {
181 0 : if (strlen(c->errbuf) > 0)
182 0 : mnstr_set_open_error(url, 0, "%s", c->errbuf);
183 : else
184 0 : mnstr_set_open_error(url, 0, "curl_easy_perform: %s", curl_easy_strerror(ret));
185 0 : curl_destroy(s);
186 0 : return NULL;
187 : }
188 0 : curl_easy_cleanup(c->handle);
189 0 : c->handle = NULL;
190 0 : c->running = 0;
191 0 : return s;
192 : }
193 :
194 : #else
195 : stream *
196 : open_urlstream(const char *url)
197 : {
198 : if (url != NULL &&
199 : strncmp(url, "file://", sizeof("file://") - 1) == 0) {
200 : url +=sizeof("file://") - 1;
201 : #ifdef _MSC_VER
202 : /* file:///C:/... -- remove third / as well */
203 : if (url[0] == '/' && url[2] == ':')
204 : url++;
205 : #endif
206 : return open_rastream(url);
207 : }
208 : mnstr_set_open_error(url, 0, "Remote URL support has been left out of this MonetDB");
209 : return NULL;
210 : }
211 : #endif /* HAVE_CURL */
|
