qLibc
qhttpclient.c
Go to the documentation of this file.
1 /******************************************************************************
2  * qLibc
3  *
4  * Copyright (c) 2010-2015 Seungyoung Kim.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions are met:
9  *
10  * 1. Redistributions of source code must retain the above copyright notice,
11  * this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright notice,
13  * this list of conditions and the following disclaimer in the documentation
14  * and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26  * POSSIBILITY OF SUCH DAMAGE.
27  *****************************************************************************/
28 
29 /**
30  * @file qhttpclient.c HTTP client object.
31  *
32  * qhttpclient implements HTTP client.
33  *
34  * Example code for simple HTTP GET operation.
35  *
36  * @code
37  * #define REMOTE_URL "/robots.txt"
38  * #define SAVEFILE "/tmp/robots.txt"
39  *
40  * int main(void) {
41  * // create new HTTP client
42  * qhttpclient_t *httpclient = qhttpclient("https://secure.qdecoder.org", 0);
43  * if(httpclient == NULL) return -1;
44  *
45  * // open file for writing
46  * int nFd = open(SAVEFILE, O_CREAT | O_TRUNC | O_WRONLY, 0644);
47  * if(nFd < 0) {
48  * httpclient->free(httpclient);
49  * return -1;
50  * }
51  *
52  * // container for storing response headers for debugging purpose
53  * qlisttbl_t *resheaders = qlisttbl(QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
54  *
55  * // download
56  * off_t nSavesize = 0;
57  * int nRescode = 0;
58  * bool bRet = httpclient->get(httpclient, REMOTE_URL, nFd, &nSavesize,
59  * &nRescode, NULL, resheaders, NULL, NULL);
60  *
61  * // close file
62  * close(nFd);
63  *
64  * // print out debugging info
65  * printf("%s %d, %d bytes saved\n", (bRet?"Success":"Failed"), nRescode,
66  * (int)nSavesize);
67  * resheaders->debug(resheaders, stdout);
68  *
69  * // de-allocate HTTP client object
70  * httpclient->free(httpclient);
71  *
72  * return (bRet ? 0 : -1);
73  * }
74  *
75  * [Output]
76  * Success 200, 30 bytes saved
77  * Date=Fri, 11 Feb 2011 23:40:50 GMT? (30)
78  * Server=Apache? (7)
79  * Last-Modified=Sun, 15 Mar 2009 11:43:07 GMT? (30)
80  * ETag="2e5c9d-1e-46526d665c8c0"? (26)
81  * Accept-Ranges=bytes? (6)
82  * Content-Length=30? (3)
83  * Cache-Control=max-age=604800? (15)
84  * Expires=Fri, 18 Feb 2011 23:40:50 GMT? (30)
85  * Connection=close? (6)
86  * Content-Type=text/plain? (11)
87  * @endcode
88  *
89  * Example code for multiple PUT operation using same keep-alive connection.
90  *
91  * @code
92  * // create new HTTP client
93  * qhttpclient_t *httpclient = qhttpclient("www.qdecoder.org", 80);
94  * if(httpclient == NULL) return;
95  *
96  * // set options
97  * httpclient->setkeepalive(httpclient, true);
98  *
99  * // make a connection
100  * if(httpclient->open(httpclient) == false) return;
101  *
102  * // upload files
103  * httpclient->put(httpclient, ...);
104  * httpclient->put(httpclient, ...); // will be done within same connection.
105  *
106  * // close connection - not necessary if we call free() just after this.
107  * httpclient->close(httpclient);
108  *
109  * // de-allocate HTTP client object
110  * httpclient->free(httpclient);
111  * @endcode
112  */
113 
114 #ifndef DISABLE_QHTTPCLIENT
115 
116 #include <stdio.h>
117 #include <stdlib.h>
118 #include <stdbool.h>
119 #include <string.h>
120 #include <unistd.h>
121 #include <fcntl.h>
122 #include <errno.h>
123 #include <sys/types.h>
124 #include <sys/socket.h>
125 #include <netinet/in.h>
126 #include <netinet/tcp.h>
127 #include <arpa/inet.h>
128 
129 #ifdef ENABLE_OPENSSL
130 #include "openssl/ssl.h"
131 #include "openssl/err.h"
132 #endif
133 
134 #include "qinternal.h"
135 #include "utilities/qio.h"
136 #include "utilities/qstring.h"
137 #include "utilities/qsocket.h"
138 #include "containers/qlisttbl.h"
139 #include "containers/qgrow.h"
140 #include "extensions/qhttpclient.h"
141 
142 #ifndef _DOXYGEN_SKIP
143 
144 static bool open_(qhttpclient_t *client);
145 static bool setssl(qhttpclient_t *client);
146 static void settimeout(qhttpclient_t *client, int timeoutms);
147 static void setkeepalive(qhttpclient_t *client, bool keepalive);
148 static void setuseragent(qhttpclient_t *client, const char *agentname);
149 
150 static bool head(qhttpclient_t *client, const char *uri, int *rescode,
151  qlisttbl_t *reqheaders, qlisttbl_t *resheaders);
152 static bool get(qhttpclient_t *client, const char *uri, int fd, off_t *savesize,
153  int *rescode, qlisttbl_t *reqheaders, qlisttbl_t *resheaders,
154  bool (*callback)(void *userdata, off_t recvbytes),
155  void *userdata);
156 static bool put(qhttpclient_t *client, const char *uri, int fd, off_t length,
157  int *rescode, qlisttbl_t *reqheaders, qlisttbl_t *resheaders,
158  bool (*callback)(void *userdata, off_t sentbytes),
159  void *userdata);
160 static void *cmd(qhttpclient_t *client, const char *method, const char *uri,
161  void *data, size_t size, int *rescode, size_t *contentslength,
162  qlisttbl_t *reqheaders, qlisttbl_t *resheaders);
163 
164 static bool sendrequest(qhttpclient_t *client, const char *method,
165  const char *uri, qlisttbl_t *reqheaders);
166 static int readresponse(qhttpclient_t *client, qlisttbl_t *resheaders,
167  off_t *contentlength);
168 
169 static ssize_t gets_(qhttpclient_t *client, char *buf, size_t bufsize);
170 static ssize_t read_(qhttpclient_t *client, void *buf, size_t nbytes);
171 static ssize_t write_(qhttpclient_t *client, const void *buf, size_t nbytes);
172 static off_t recvfile(qhttpclient_t *client, int fd, off_t nbytes);
173 static off_t sendfile_(qhttpclient_t *client, int fd, off_t nbytes);
174 
175 static bool _close(qhttpclient_t *client);
176 static void _free(qhttpclient_t *client);
177 
178 // internal usages
179 static bool _set_socket_option(int socket);
180 static bool _parse_uri(const char *uri, bool *protocol, char *hostname,
181  size_t namesize, int *port);
182 
183 #endif
184 
185 //
186 // HTTP RESPONSE CODE
187 //
188 #define HTTP_NO_RESPONSE (0)
189 #define HTTP_CODE_CONTINUE (100)
190 #define HTTP_CODE_OK (200)
191 #define HTTP_CODE_CREATED (201)
192 #define HTTP_CODE_NO_CONTENT (204)
193 #define HTTP_CODE_MULTI_STATUS (207)
194 #define HTTP_CODE_MOVED_TEMPORARILY (302)
195 #define HTTP_CODE_NOT_MODIFIED (304)
196 #define HTTP_CODE_BAD_REQUEST (400)
197 #define HTTP_CODE_FORBIDDEN (403)
198 #define HTTP_CODE_NOT_FOUND (404)
199 #define HTTP_CODE_METHOD_NOT_ALLOWED (405)
200 #define HTTP_CODE_REQUEST_TIME_OUT (408)
201 #define HTTP_CODE_REQUEST_URI_TOO_LONG (414)
202 #define HTTP_CODE_INTERNAL_SERVER_ERROR (500)
203 #define HTTP_CODE_NOT_IMPLEMENTED (501)
204 #define HTTP_CODE_SERVICE_UNAVAILABLE (503)
205 
206 #define HTTP_PROTOCOL_11 "HTTP/1.1"
207 
208 //
209 // TCP SOCKET DEFINITION
210 //
211 #define SET_TCP_LINGER_TIMEOUT (15) /*< linger seconds, 0 for disable */
212 #define SET_TCP_NODELAY (1) /*< 0 for disable */
213 #define MAX_SHUTDOWN_WAIT (100) /*< maximum shutdown wait, unit is ms */
214 #define MAX_ATOMIC_DATA_SIZE (32 * 1024) /*< maximum sending bytes */
215 
216 #ifdef ENABLE_OPENSSL
217 struct SslConn {
218  SSL *ssl;
219  SSL_CTX *ctx;
220 };
221 #endif
222 
223 /**
224  * Initialize & create new HTTP client.
225  *
226  * @param destname remote address, one of IP address, FQDN domain name and URI.
227  * @param port remote port number. (can be 0 when destname is URI)
228  *
229  * @return HTTP client object if succcessful, otherwise returns NULL.
230  *
231  * @code
232  * qhttpclient_t *client = qhttpclient("1.2.3.4", 80);
233  * qhttpclient_t *client = qhttpclient("www.qdecoder.org", 80);
234  * qhttpclient_t *client = qhttpclient("http://www.qdecoder.org", 0);
235  * qhttpclient_t *client = qhttpclient("http://www.qdecoder.org:80", 0);
236  * qhttpclient_t *client = qhttpclient("https://www.qdecoder.org", 0);
237  * qhttpclient_t *client = qhttpclient("https://www.qdecoder.org:443", 0);
238  * @endcode
239  *
240  * @note
241  * Keep-alive feature is turned off by default. Turn it on by calling
242  * setkeepalive(). If destname is URI string starting with
243  * "https://", setssl() will be called internally.
244  */
245 qhttpclient_t *qhttpclient(const char *destname, int port) {
246  bool ishttps = false;
247  char hostname[256];
248  if (port == 0 || strstr(hostname, "://") != NULL) {
249  if (_parse_uri(destname, &ishttps, hostname, sizeof(hostname), &port)
250  == false) {
251  DEBUG("Can't parse URI %s", destname);
252  return NULL;
253  }
254 
255  DEBUG("https: %d, hostname: %s, port:%d\n", ishttps, hostname, port);
256  } else {
257  qstrcpy(hostname, sizeof(hostname), destname);
258  }
259 
260  // get remote address
261  struct sockaddr_in addr;
262  if (qsocket_get_addr(&addr, hostname, port) == false) {
263  return NULL;
264  }
265 
266  // allocate object
267  qhttpclient_t *client = (qhttpclient_t *) malloc(sizeof(qhttpclient_t));
268  if (client == NULL)
269  return NULL;
270  memset((void *) client, 0, sizeof(qhttpclient_t));
271 
272  // initialize object
273  client->socket = -1;
274 
275  memcpy((void *) &client->addr, (void *) &addr, sizeof(client->addr));
276  client->hostname = strdup(hostname);
277  client->port = port;
278 
279  // member methods
280  client->setssl = setssl;
281  client->settimeout = settimeout;
282  client->setkeepalive = setkeepalive;
283  client->setuseragent = setuseragent;
284 
285  client->open = open_;
286 
287  client->head = head;
288  client->get = get;
289  client->put = put;
290  client->cmd = cmd;
291 
292  client->sendrequest = sendrequest;
293  client->readresponse = readresponse;
294 
295  client->gets = gets_;
296  client->read = read_;
297  client->write = write_;
298  client->recvfile = recvfile;
299  client->sendfile = sendfile_;
300 
301  client->close = _close;
302  client->free = _free;
303 
304  // init client
305  settimeout(client, 0);
306  setkeepalive(client, false);
307  setuseragent(client, QHTTPCLIENT_NAME);
308  if (ishttps == true)
309  setssl(client);
310 
311  return client;
312 }
313 
314 /**
315  * qhttpclient->setssl(): Sets connection to HTTPS connection
316  *
317  * @param client qhttpclient object pointer
318  *
319  * @code
320  * httpclient->setssl(httpclient);
321  * @endcode
322  */
323 static bool setssl(qhttpclient_t *client) {
324 #ifdef ENABLE_OPENSSL
325  static bool initialized = false;
326 
327  if (client->socket >= 0) {
328  // must be set before making a connection.
329  return false;
330  }
331 
332  // init openssl
333  if (initialized == false) {
334  initialized = true;
335  SSL_load_error_strings();
336  SSL_library_init();
337  }
338 
339  // allocate ssl structure
340  if (client->ssl == NULL) {
341  client->ssl = malloc(sizeof(struct SslConn));
342  if (client->ssl == NULL) return false;
343  memset(client->ssl, 0, sizeof(struct SslConn));
344  }
345 
346  return true;
347 #else
348  return false;
349 #endif
350 }
351 
352 /**
353  * qhttpclient->settimeout(): Sets connection wait timeout.
354  *
355  * @param client qhttpclient object pointer
356  * @param timeoutms timeout mili-seconds. 0 for system defaults
357  *
358  * @code
359  * httpclient->settimeout(httpclient, 0); // default
360  * httpclient->settimeout(httpclient, 5000); // 5 seconds
361  * @endcode
362  */
363 static void settimeout(qhttpclient_t *client, int timeoutms) {
364  if (timeoutms <= 0)
365  timeoutms = -1;
366  client->timeoutms = timeoutms;
367 }
368 
369 /**
370  * qhttpclient->setkeepalive(): Sets KEEP-ALIVE feature on/off.
371  *
372  * @param client qhttpclient object pointer
373  * @param keepalive true to set keep-alive on, false to set keep-alive off
374  *
375  * @code
376  * httpclient->setkeepalive(httpclient, true); // keep-alive on
377  * httpclient->setkeepalive(httpclient, false); // keep-alive off
378  * @endcode
379  */
380 static void setkeepalive(qhttpclient_t *client, bool keepalive) {
381  client->keepalive = keepalive;
382 }
383 
384 /**
385  * qhttpclient->setuseragent(): Sets user-agent string.
386  *
387  * @param client qhttpclient object pointer
388  * @param useragent user-agent string
389  *
390  * @code
391  * httpclient->setuseragent(httpclient, "MyAgent/1.0");
392  * @endcode
393  */
394 static void setuseragent(qhttpclient_t *client, const char *useragent) {
395  if (client->useragent != NULL)
396  free(client->useragent);
397  client->useragent = strdup(useragent);
398 }
399 
400 /**
401  * qhttpclient->open(): Opens a connection to the remote host.
402  *
403  * @param client qhttpclient object pointer
404  *
405  * @return true if successful, otherwise returns false
406  *
407  * @note
408  * Don't need to open a connection unless you definitely need to do this,
409  * because qhttpclient open a connection automatically when it's needed.
410  * This function also can be used to veryfy a connection failure with remote
411  * host.
412  *
413  * @code
414  * if(httpclient->open(httpclient) == false) return;
415  * @endcode
416  */
417 static bool open_(qhttpclient_t *client) {
418  if (client->socket >= 0) {
419  // check if connection is still alive
420  if (qio_wait_writable(client->socket, 0) > 0)
421  return true;
422  _close(client);
423  }
424 
425  // create new socket
426  int sockfd = socket(AF_INET, SOCK_STREAM, 0);
427  if (sockfd < 0) {
428  DEBUG("sockfd creation failed.");
429  return false;
430  }
431 
432  // set to non-block socket if timeout is set
433  int sockflag = 0;
434  if (client->timeoutms > 0) {
435  sockflag = fcntl(sockfd, F_GETFL, 0);
436  fcntl(sockfd, F_SETFL, sockflag | O_NONBLOCK);
437  }
438 
439  // try to connect
440  int status = connect(sockfd, (struct sockaddr *) &client->addr,
441  sizeof(client->addr));
442  if (status < 0
443  && (errno != EINPROGRESS
444  || qio_wait_writable(sockfd, client->timeoutms) <= 0)) {
445  DEBUG("connection failed. (%d)", errno);
446  close(sockfd);
447  return false;
448  }
449 
450  // restore to block socket
451  if (client->timeoutms > 0) {
452  fcntl(sockfd, F_SETFL, sockflag);
453  }
454 
455  // store socket descriptor
456  client->socket = sockfd;
457 
458  // set socket option
459  _set_socket_option(sockfd);
460 
461 #ifdef ENABLE_OPENSSL
462  // set SSL option
463  if (client->ssl != NULL) {
464  // get ssl context using SSL 2 or 3
465  struct SslConn *ssl = client->ssl;
466  ssl->ctx = SSL_CTX_new(SSLv23_client_method());
467  if (ssl->ctx == NULL) {
468  DEBUG("OpenSSL: %s", ERR_reason_error_string(ERR_get_error()));
469  _close(client);
470  return false;
471  }
472 
473  // get ssl handle
474  ssl->ssl = SSL_new(ssl->ctx);
475  if (ssl->ssl == NULL) {
476  DEBUG("OpenSSL: %s", ERR_reason_error_string(ERR_get_error()));
477  _close(client);
478  return false;
479  }
480 
481  // map ssl handle with socket
482  if (SSL_set_fd(ssl->ssl, client->socket) != 1) {
483  DEBUG("OpenSSL: %s", ERR_reason_error_string(ERR_get_error()));
484  _close(client);
485  return false;
486  }
487 
488  // set options
489  SSL_set_connect_state(ssl->ssl);
490 
491  // handshake
492  if (SSL_connect(ssl->ssl) != 1) {
493  DEBUG("OpenSSL: %s", ERR_reason_error_string(ERR_get_error()));
494  _close(client);
495  return false;
496  }
497 
498  DEBUG("ssl initialized");
499  }
500 #endif
501 
502  return true;
503 }
504 
505 /**
506  * qhttpclient->head(): Sends a HEAD request.
507  *
508  * @param client qhttpclient object pointer.
509  * @param uri URL encoded remote URI for downloading file.
510  * ("/path" or "http://.../path")
511  * @param rescode if not NULL, remote response code will be stored.
512  * (can be NULL)
513  * @param reqheaders qlisttbl_t pointer which contains additional user
514  * request headers. (can be NULL)
515  * @param resheaders qlisttbl_t pointer for storing response headers.
516  * (can be NULL)
517  *
518  * @return true if successful(got 200 response), otherwise returns false
519  *
520  * @code
521  * main() {
522  * // create new HTTP client
523  * qhttpclient_t *httpclient = qhttpclient("http://www.qdecoder.org", 0);
524  * if(httpclient == NULL) return;
525  *
526  * // set additional custom headers
527  * qlisttbl_t *reqheaders = qlisttbl(QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
528  * qlisttbl_t *resheaders = qlisttbl(QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
529  *
530  * // send HEAD request
531  * int nRescode = 0;
532  * char *pszEncPath = qEncodeUrl("/img/qdecoder.png");
533  * bool bRet = httpclient->head(httpclient, pszEncPath, &nRescode,
534  * reqheaders, resheaders);
535  * free(pszEncPath);
536  *
537  * // to print out request, response headers
538  * reqheaders->debug(reqheaders, stdout);
539  * resheaders->debug(resheaders, stdout);
540  *
541  * // check results
542  * if(bRet == false) {
543  * ...(error occured)...
544  * }
545  *
546  * // free resources
547  * httpclient->free(httpclient);
548  * reqheaders->free(reqheaders);
549  * resheaders->free(resheaders);
550  * }
551  * @endcode
552  */
553 static bool head(qhttpclient_t *client, const char *uri, int *rescode,
554  qlisttbl_t *reqheaders, qlisttbl_t *resheaders) {
555 
556  // reset rescode
557  if (rescode != NULL)
558  *rescode = 0;
559 
560  // generate request headers if necessary
561  bool freeReqHeaders = false;
562  if (reqheaders == NULL) {
563  reqheaders = qlisttbl(
564  QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
565  freeReqHeaders = true;
566  }
567 
568  // add additional headers
569  reqheaders->putstr(reqheaders, "Accept", "*/*");
570 
571  // send request
572  bool sendret = sendrequest(client, "HEAD", uri, reqheaders);
573  if (freeReqHeaders == true)
574  reqheaders->free(reqheaders);
575  if (sendret == false) {
576  _close(client);
577  return false;
578  }
579 
580  // read response
581  off_t clength = 0;
582  int resno = readresponse(client, resheaders, &clength);
583  if (rescode != NULL)
584  *rescode = resno;
585 
586  // throw out content
587  if (clength > 0) {
588  if (read_(client, NULL, clength) != clength) {
589  _close(client);
590  }
591  }
592 
593  // close connection if required
594  if (client->keepalive == false || client->connclose == true) {
595  _close(client);
596  }
597 
598  if (resno == HTTP_CODE_OK)
599  return true;
600  return false;
601 }
602 
603 /**
604  * qhttpclient->get(): Downloads a file from the remote host using GET
605  * method.
606  *
607  * @param client qhttpclient object pointer.
608  * @param uri URL encoded remote URI for downloading file.
609  * ("/path" or "http://.../path")
610  * @param fd opened file descriptor for writing.
611  * @param savesize if not NULL, the length of stored bytes will be stored.
612  * (can be NULL)
613  * @param rescode if not NULL, remote response code will be stored.
614  * (can be NULL)
615  * @param reqheaders qlisttbl_t pointer which contains additional user
616  * request headers. (can be NULL)
617  * @param resheaders qlisttbl_t pointer for storing response headers.
618  * (can be NULL)
619  * @param callback set user call-back function. (can be NULL)
620  * @param userdata set user data for call-back. (can be NULL)
621  *
622  * @return true if successful(200 OK), otherwise returns false
623  *
624  * @code
625  * struct userdata {
626  * ...
627  * };
628  *
629  * static bool callback(void *userdata, off_t sentbytes) {
630  * struct userdata *pMydata = (struct userdata*)userdata;
631  * ...(codes)...
632  * if(need_to_cancel) return false; // stop file uploading immediately
633  * return true;
634  * }
635  *
636  * main() {
637  * // create new HTTP client
638  * qhttpclient_t *httpclient = qhttpclient("http://www.qdecoder.org", 0);
639  * if(httpclient == NULL) return;
640  *
641  * // open file
642  * int nFd = open("/tmp/test.data", O_WRONLY | O_CREAT, 0644);
643  *
644  * // set additional custom headers
645  * qlisttbl_t *reqheaders = qlisttbl(QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
646  * qlisttbl_t *resheaders = qlisttbl(QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
647  *
648  * // set userdata
649  * struct userdata mydata;
650  * ...(codes)...
651  *
652  * // send file
653  * int nRescode = 0;
654  * off_t nSavesize = 0;
655  * char *pszEncPath = qEncodeUrl("/img/qdecoder.png");
656  * bool bRet = httpclient->get(httpclient, pszEncPath, nFd, &nSavesize,
657  * &nRescode,
658  * reqheaders, resheaders,
659  * callback, (void*)&mydata);
660  * free(pszEncPath);
661  *
662  * // to print out request, response headers
663  * reqheaders->debug(reqheaders, stdout);
664  * resheaders->debug(resheaders, stdout);
665  *
666  * // check results
667  * if(bRet == false) {
668  * ...(error occured)...
669  * }
670  *
671  * // free resources
672  * httpclient->free(httpclient);
673  * reqheaders->free(reqheaders);
674  * resheaders->free(resheaders);
675  * close(nFd);
676  * }
677  * @endcode
678  *
679  * @note
680  * The call-back function will be called peridically whenever it send data as
681  * much as MAX_ATOMIC_DATA_SIZE. To stop uploading, return false in the
682  * call-back function, then PUT process will be stopped immediately.
683  * If a connection was not opened, it will open a connection automatically.
684  *
685  * @note
686  * The "rescode" will be set if it received any response code from a remote
687  * server even though it returns false.
688  */
689 static bool get(qhttpclient_t *client, const char *uri, int fd, off_t *savesize,
690  int *rescode, qlisttbl_t *reqheaders, qlisttbl_t *resheaders,
691  bool (*callback)(void *userdata, off_t recvbytes),
692  void *userdata) {
693 
694  // reset rescode
695  if (rescode != NULL)
696  *rescode = 0;
697  if (savesize != NULL)
698  *savesize = 0;
699 
700  // generate request headers if necessary
701  bool freeReqHeaders = false;
702  if (reqheaders == NULL) {
703  reqheaders = qlisttbl(
704  QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
705  freeReqHeaders = true;
706  }
707 
708  // add additional headers
709  reqheaders->putstr(reqheaders, "Accept", "*/*");
710 
711  // send request
712  bool sendret = sendrequest(client, "GET", uri, reqheaders);
713  if (freeReqHeaders == true)
714  reqheaders->free(reqheaders);
715  if (sendret == false) {
716  _close(client);
717  return false;
718  }
719 
720  // read response
721  off_t clength = 0;
722  int resno = readresponse(client, resheaders, &clength);
723  if (rescode != NULL)
724  *rescode = resno;
725 
726  // check response code
727  if (resno != HTTP_CODE_OK) {
728  // throw out content
729  if (clength > 0) {
730  if (read_(client, NULL, clength) != clength) {
731  _close(client);
732  }
733  }
734 
735  // close connection if required
736  if (client->keepalive == false || client->connclose == true) {
737  _close(client);
738  }
739  return false;
740  }
741 
742  // start retrieving data
743  off_t recv = 0;
744  if (callback != NULL && callback(userdata, recv) == false) {
745  _close(client);
746  return false;
747  }
748 
749  if (clength > 0) {
750  while (recv < clength) {
751  unsigned int recvsize; // this time receive size
752  if (clength - recv < MAX_ATOMIC_DATA_SIZE) {
753  recvsize = clength - recv;
754  } else {
755  recvsize = MAX_ATOMIC_DATA_SIZE;
756  }
757 
758  ssize_t ret = recvfile(client, fd, recvsize);
759  if (ret <= 0)
760  break; // Connection closed by peer
761  recv += ret;
762  if (savesize != NULL)
763  *savesize = recv;
764 
765  if (callback != NULL) {
766  if (callback(userdata, recv) == false) {
767  _close(client);
768  return false;
769  }
770  }
771  }
772 
773  if (recv != clength) {
774  _close(client);
775  return false;
776  }
777 
778  } else if (clength == -1) { // chunked
779  bool completed = false;
780  do {
781  // read chunk size
782  char buf[64];
783  if (gets_(client, buf, sizeof(buf)) <= 0)
784  break;
785 
786  // parse chunk size
787  unsigned int recvsize; // this time chunk size
788  sscanf(buf, "%x", &recvsize);
789  if (recvsize == 0) {
790  // end of transfer
791  completed = true;
792  }
793 
794  // save chunk
795  if (recvsize > 0) {
796  ssize_t ret = recvfile(client, fd, recvsize);
797  if (ret != recvsize)
798  break;
799  recv += ret;
800  DEBUG("%zd %zd", recv, ret);
801  if (savesize != NULL)
802  *savesize = recv;
803  }
804 
805  // read tailing CRLF
806  if (gets_(client, buf, sizeof(buf)) <= 0)
807  break;
808 
809  // call back
810  if (recvsize > 0 && callback != NULL
811  && callback(userdata, recv) == false) {
812  _close(client);
813  return false;
814  }
815  } while (completed == false);
816 
817  if (completed == false) {
818  DEBUG("Broken pipe. %jd/chunked, errno=%d", recv, errno);
819  _close(client);
820  return false;
821  }
822  }
823 
824  // close connection
825  if (client->keepalive == false || client->connclose == true) {
826  _close(client);
827  }
828 
829  return true;
830 }
831 
832 /**
833  * qhttpclient->put(): Uploads a file to the remote host using PUT method.
834  *
835  * @param client qhttpclient object pointer.
836  * @param uri remote URL for uploading file.
837  * ("/path" or "http://.../path")
838  * @param fd opened file descriptor for reading.
839  * @param length send size.
840  * @param rescode if not NULL, remote response code will be stored.
841  * (can be NULL)
842  * @param reqheaders qlisttbl_t pointer which contains additional user
843  * request headers. (can be NULL)
844  * @param resheaders qlisttbl_t pointer for storing response headers.
845  * (can be NULL)
846  * @param callback set user call-back function. (can be NULL)
847  * @param userdata set user data for call-back. (can be NULL)
848  *
849  * @return true if successful(201 Created), otherwise returns false
850  *
851  * @code
852  * struct userdata {
853  * ...
854  * };
855  *
856  * static bool callback(void *userdata, off_t sentbytes) {
857  * struct userdata *pMydata = (struct userdata*)userdata;
858  * ...(codes)...
859  * if(need_to_cancel) return false; // stop file uploading immediately
860  * return true;
861  * }
862  *
863  * main() {
864  * // create new HTTP client
865  * qhttpclient_t *httpclient = qhttpclient("http://www.qdecoder.org", 0);
866  * if(httpclient == NULL) return;
867  *
868  * // open file
869  * int nFd = open(...);
870  * off_t nFileSize = ...;
871  * char *pFileMd5sum = ...;
872  * time_t nFileDate = ...;
873  *
874  * // set additional custom headers
875  * qlisttbl_t *reqheaders = qlisttbl(QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
876  * reqheaders->putstr(reqheaders, "X-FILE-MD5SUM", pFileMd5sum);
877  * reqheaders->putInt(reqheaders, "X-FILE-DATE", nFileDate);
878  *
879  * // set userdata
880  * struct userdata mydata;
881  * ...(codes)...
882  *
883  * // send file
884  * int nRescode = 0;
885  * qlisttbl_t *resheaders = qlisttbl(QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
886  * bool bRet = httpclient->put(httpclient,
887  * "/img/qdecoder.png", nFd, nFileSize,
888  * &nRescode,
889  * reqheaders, resheaders,
890  * callback, (void*)&mydata);
891  * // to print out request, response headers
892  * reqheaders->debug(reqheaders, stdout);
893  * resheaders->debug(resheaders, stdout);
894  *
895  * // check results
896  * if(bRet == false) {
897  * ...(error occured)...
898  * }
899  *
900  * // free resources
901  * httpclient->free(httpclient);
902  * reqheaders->free(reqheaders);
903  * resheaders->free(resheaders);
904  * close(nFd);
905  * }
906  * @endcode
907  *
908  * @note
909  * The call-back function will be called peridically whenever it send data as
910  * much as MAX_ATOMIC_DATA_SIZE. To stop uploading, return false in the
911  * call-back function, then PUT process will be stopped immediately.
912  * If a connection was not opened, it will open a connection automatically.
913  *
914  * @note
915  * The "rescode" will be set if it received any response code from a remote
916  * server even though it returns false.
917  */
918 static bool put(qhttpclient_t *client, const char *uri, int fd, off_t length,
919  int *rescode, qlisttbl_t *reqheaders, qlisttbl_t *resheaders,
920  bool (*callback)(void *userdata, off_t sentbytes),
921  void *userdata) {
922 
923  // reset rescode
924  if (rescode != NULL)
925  *rescode = 0;
926 
927  // generate request headers
928  bool freeReqHeaders = false;
929  if (reqheaders == NULL) {
930  reqheaders = qlisttbl(
931  QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
932  freeReqHeaders = true;
933  }
934 
935  // add additional headers
936  reqheaders->putstrf(reqheaders, "Content-Length", "%jd", length);
937  reqheaders->putstr(reqheaders, "Expect", "100-continue");
938 
939  // send request
940  bool sendret = sendrequest(client, "PUT", uri, reqheaders);
941  if (freeReqHeaders == true) {
942  reqheaders->free(reqheaders);
943  reqheaders = NULL;
944  }
945  if (sendret == false) {
946  _close(client);
947  return false;
948  }
949 
950  // wait 100-continue
951  if (qio_wait_readable(client->socket, client->timeoutms) <= 0) {
952  DEBUG("timed out %d", client->timeoutms);
953  _close(client);
954  return false;
955  }
956 
957  // read response
958  off_t clength = 0;
959  int resno = readresponse(client, resheaders, &clength);
960  if (resno != HTTP_CODE_CONTINUE) {
961  if (rescode != NULL)
962  *rescode = resno;
963 
964  if (clength > 0) {
965  if (read_(client, NULL, clength) != clength) {
966  _close(client);
967  }
968  }
969 
970  // close connection if required
971  if (client->keepalive == false || client->connclose == true) {
972  _close(client);
973  }
974  return false;
975  }
976 
977  // send data
978  off_t sent = 0;
979  if (callback != NULL) {
980  if (callback(userdata, sent) == false) {
981  _close(client);
982  return false;
983  }
984  }
985  if (length > 0) {
986  while (sent < length) {
987  size_t sendsize; // this time sending size
988  if (length - sent < MAX_ATOMIC_DATA_SIZE)
989  sendsize = length - sent;
990  else
991  sendsize = MAX_ATOMIC_DATA_SIZE;
992 
993  ssize_t ret = sendfile_(client, fd, sendsize);
994  if (ret <= 0)
995  break; // Connection closed by peer
996  sent += ret;
997 
998  if (callback != NULL) {
999  if (callback(userdata, sent) == false) {
1000  _close(client);
1001  return false;
1002  }
1003  }
1004  }
1005 
1006  if (sent != length) {
1007  _close(client);
1008  return false;
1009  }
1010 
1011  if (callback != NULL) {
1012  if (callback(userdata, sent) == false) {
1013  _close(client);
1014  return false;
1015  }
1016  }
1017  }
1018 
1019  // read response
1020  clength = 0;
1021  resno = readresponse(client, resheaders, &clength);
1022  if (rescode != NULL)
1023  *rescode = resno;
1024 
1025  if (resno == HTTP_NO_RESPONSE) {
1026  _close(client);
1027  return false;
1028  }
1029 
1030  if (clength > 0) {
1031  if (read_(client, NULL, clength) != clength) {
1032  _close(client);
1033  }
1034  }
1035 
1036  // close connection
1037  if (client->keepalive == false || client->connclose == true) {
1038  _close(client);
1039  }
1040 
1041  if (resno == HTTP_CODE_CREATED)
1042  return true;
1043  return false;
1044 }
1045 
1046 /**
1047  * qhttpclient->cmd(): Sends a custom request(method) to the remote host
1048  * and reads it's response.
1049  *
1050  * @param client qhttpclient object pointer.
1051  * @param method method name.
1052  * @param uri remote URL for uploading file.
1053  * ("/path" or "http://.../path")
1054  * @param data data to send. (can be NULL)
1055  * @param size data size.
1056  * @param rescode if not NULL, remote response code will be stored.
1057  * (can be NULL)
1058  * @param contentslength if not NULL, the contents length will be stored.
1059  * (can be NULL)
1060  * @param reqheaders qlisttbl_t pointer which contains additional user
1061  * request headers. (can be NULL)
1062  * @param resheaders qlisttbl_t pointer for storing response headers.
1063  * (can be NULL)
1064  *
1065  * @return malloced content data if successful, otherwise returns NULL
1066  *
1067  * @code
1068  * int nResCode;
1069  * size_t nContentsLength;
1070  * void *contents = httpclient->cmd(httpclient, "DELETE" "/img/qdecoder.png",
1071  * NULL, 0
1072  * &nRescode, &nContentsLength
1073  * NULL, NULL);
1074  * if(contents == NULL) {
1075  * ...(error occured)...
1076  * } else {
1077  * printf("Response code : %d\n", nResCode);
1078  * printf("Contents length : %zu\n", nContentsLength);
1079  * printf("Contents : %s\n", (char*)contents); // if contents is printable
1080  * free(contents); // de-allocate
1081  * }
1082  * @endcode
1083  *
1084  * @note
1085  * This store server's response into memory so if you expect server responses
1086  * large amount of data, consider to use sendrequest() and readresponse()
1087  * instead of using this. The returning malloced content will be allocated
1088  * +1 byte than actual content size 'contentslength' and will be null
1089  * terminated.
1090  */
1091 static void *cmd(qhttpclient_t *client, const char *method, const char *uri,
1092  void *data, size_t size, int *rescode, size_t *contentslength,
1093  qlisttbl_t *reqheaders, qlisttbl_t *resheaders) {
1094 
1095  // reset rescode
1096  if (rescode != NULL)
1097  *rescode = 0;
1098  if (contentslength != NULL)
1099  *contentslength = 0;
1100 
1101  // send request
1102  bool freeReqHeaders = false;
1103  if (reqheaders == NULL && data != NULL && size > 0) {
1104  reqheaders = qlisttbl(
1105  QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
1106  reqheaders->putstrf(reqheaders, "Content-Length", "%jd", size);
1107  freeReqHeaders = true;
1108  }
1109 
1110  bool sendret = sendrequest(client, method, uri, reqheaders);
1111  if (freeReqHeaders == true) {
1112  reqheaders->free(reqheaders);
1113  reqheaders = NULL;
1114  }
1115  if (sendret == false) {
1116  _close(client);
1117  return NULL;
1118  }
1119 
1120  // send data
1121  if (data != NULL && size > 0) {
1122  ssize_t written = write_(client, data, size);
1123  if (written != size) {
1124  _close(client);
1125  return NULL;
1126  }
1127  }
1128 
1129  // read response
1130  off_t clength = 0;
1131  int resno = readresponse(client, resheaders, &clength);
1132  if (rescode != NULL)
1133  *rescode = resno;
1134  if (contentslength != NULL)
1135  *contentslength = clength;
1136 
1137  // malloc data
1138  void *content = NULL;
1139  if (clength > 0) {
1140  content = malloc(clength + 1);
1141  if (content != NULL) {
1142  if (read_(client, content, clength) == clength) {
1143  *(char *) (content + clength) = '\0';
1144  } else {
1145  free(content);
1146  content = NULL;
1147  _close(client);
1148  }
1149  }
1150  } else {
1151  // succeed. to distinguish between ok and error
1152  content = strdup("");
1153  }
1154 
1155  // close connection
1156  if (client->keepalive == false || client->connclose == true) {
1157  _close(client);
1158  }
1159 
1160  return content;
1161 }
1162 
1163 /**
1164  * qhttpclient->sendrequest(): Sends a HTTP request to the remote host.
1165  *
1166  * @param client qhttpclient object pointer
1167  * @param method HTTP method name
1168  * @param uri URI string for the method. ("/path" or "http://.../path")
1169  * @param reqheaders qlisttbl_t pointer which contains additional user
1170  * request headers. (can be NULL)
1171  *
1172  * @return true if successful, otherwise returns false
1173  *
1174  * @note
1175  * Default headers(Host, User-Agent, Connection) will be used if reqheaders
1176  * does not have those headers in it.
1177  *
1178  * @code
1179  * qlisttbl_t *reqheaders = qlisttbl(QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
1180  * reqheaders->putstr(reqheaders, "Date", qTimeGetGmtStaticStr(0), true);
1181  *
1182  * httpclient->sendrequest(client,
1183  * "DELETE", "/img/qdecoder.png", reqheaders);
1184  * @endcode
1185  */
1186 static bool sendrequest(qhttpclient_t *client, const char *method,
1187  const char *uri, qlisttbl_t *reqheaders) {
1188  if (open_(client) == false) {
1189  return false;
1190  }
1191 
1192  // generate request headers if necessary
1193  bool freeReqHeaders = false;
1194  if (reqheaders == NULL) {
1195  reqheaders = qlisttbl(
1196  QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
1197  if (reqheaders == NULL)
1198  return false;
1199  freeReqHeaders = true;
1200  }
1201 
1202  // append default headers
1203  if (reqheaders->get(reqheaders, "Host", NULL, false) == NULL) {
1204  reqheaders->putstrf(reqheaders, "Host", "%s:%d", client->hostname,
1205  client->port);
1206  }
1207  if (reqheaders->get(reqheaders, "User-Agent", NULL, false) == NULL) {
1208  reqheaders->putstr(reqheaders, "User-Agent", client->useragent);
1209  }
1210  if (reqheaders->get(reqheaders, "Connection", NULL, false) == NULL) {
1211  reqheaders->putstr(
1212  reqheaders, "Connection",
1213  (client->keepalive == true) ? "Keep-Alive" : "close");
1214  }
1215 
1216  // create stream buffer
1217  qgrow_t *outBuf = qgrow(0);
1218  if (outBuf == NULL)
1219  return false;
1220 
1221  // buffer out command
1222  outBuf->addstrf(outBuf, "%s %s %s\r\n", method, uri,
1223  HTTP_PROTOCOL_11);
1224 
1225  // buffer out headers
1226  qlisttbl_obj_t obj;
1227  memset((void *) &obj, 0, sizeof(obj)); // must be cleared before call
1228  reqheaders->lock(reqheaders);
1229  while (reqheaders->getnext(reqheaders, &obj, NULL, false) == true) {
1230  outBuf->addstrf(outBuf, "%s: %s\r\n", obj.name, (char *) obj.data);
1231  }
1232  reqheaders->unlock(reqheaders);
1233 
1234  outBuf->addstrf(outBuf, "\r\n");
1235 
1236  // stream out
1237  size_t towrite = 0;
1238  char *final = outBuf->toarray(outBuf, &towrite);
1239  ssize_t written = 0;
1240  if (final != NULL) {
1241  written = write_(client, final, towrite);
1242  free(final);
1243  }
1244 
1245  // de-allocate
1246  outBuf->free(outBuf);
1247  if (freeReqHeaders == true)
1248  reqheaders->free(reqheaders);
1249 
1250  if (written > 0 && written == towrite)
1251  return true;
1252  return false;
1253 }
1254 
1255 /**
1256  * qhttpclient->readresponse(): Reads HTTP response header from the
1257  * remote host.
1258  *
1259  * @param client qhttpclient object pointer
1260  * @param resheaders qlisttbl_t pointer for storing response headers.
1261  * (can be NULL)
1262  * @param contentlength length of content body(or -1 for chunked transfer
1263  * encoding) will be stored. (can be NULL)
1264  *
1265  * @return numeric HTTP response code if successful, otherwise returns 0.
1266  *
1267  * @code
1268  * // send request
1269  * httpclient->sendrequest(client, "DELETE", "/img/qdecoder.png", NULL);
1270  *
1271  * // read response
1272  * qlisttbl_t *resheaders = qlisttbl(QLISTTBL_UNIQUE | QLISTTBL_CASEINSENSITIVE);
1273  * off_t clength;
1274  * int rescode = httpclient->readresponse(client, resheaders, &clength);
1275  * if(clength > 0) {
1276  * // read & throw out a content. don't need content
1277  * httpclient->read(client, NULL, clength);
1278  * }
1279  * @endcode
1280  *
1281  * @note
1282  * Data of content body must be read by a application side, if you want to use
1283  * Keep-Alive session. Please refer qhttpclient->read().
1284  */
1285 static int readresponse(qhttpclient_t *client, qlisttbl_t *resheaders,
1286  off_t *contentlength) {
1287  if (contentlength != NULL) {
1288  *contentlength = 0;
1289  }
1290 
1291  // read response
1292  char buf[1024];
1293  if (gets_(client, buf, sizeof(buf)) <= 0)
1294  return HTTP_NO_RESPONSE;
1295 
1296  // parse response code
1297  if (strncmp(buf, "HTTP/", CONST_STRLEN("HTTP/")))
1298  return HTTP_NO_RESPONSE;
1299  char *tmp = strstr(buf, " ");
1300  if (tmp == NULL)
1301  return HTTP_NO_RESPONSE;
1302  int rescode = atoi(tmp + 1);
1303  if (rescode == 0)
1304  return HTTP_NO_RESPONSE;
1305 
1306  // read headers
1307  while (gets_(client, buf, sizeof(buf)) > 0) {
1308  if (buf[0] == '\0')
1309  break;
1310 
1311  // parse header
1312  char *name = buf;
1313  char *value = strstr(buf, ":");
1314  if (value != NULL) {
1315  *value = '\0';
1316  value += 1;
1317  qstrtrim(value);
1318  } else {
1319  // missing colon
1320  value = "";
1321  }
1322 
1323  if (resheaders != NULL) {
1324  resheaders->putstr(resheaders, name, value);
1325  }
1326 
1327  // check Connection header
1328  if (!strcasecmp(name, "Connection")) {
1329  if (!strcasecmp(value, "close")) {
1330  client->connclose = true;
1331  }
1332  }
1333  // check Content-Length & Transfer-Encoding header
1334  else if (contentlength != NULL && *contentlength == 0) {
1335  if (!strcasecmp(name, "Content-Length")) {
1336  *contentlength = atoll(value);
1337  }
1338  // check transfer-encoding header
1339  else if (!strcasecmp(name, "Transfer-Encoding")
1340  && !strcasecmp(value, "chunked")) {
1341  *contentlength = -1;
1342  }
1343  }
1344  }
1345 
1346  return rescode;
1347 }
1348 
1349 /**
1350  * qhttpclient->gets(): Reads a text line from a HTTP/HTTPS stream.
1351  *
1352  * @param client qhttpclient object pointer
1353  * @param buf data buffer pointer
1354  * @param bufsize buffer size
1355  *
1356  * @return the number of bytes read from file descriptor if successful,
1357  * otherwise returns -1.
1358  *
1359  * @note
1360  * Be sure the return value does not mean the length of actual stored data.
1361  * It means how many bytes are read from the file descriptor, so the new-line
1362  * characters will be counted, but not stored.
1363  */
1364 static ssize_t gets_(qhttpclient_t *client, char *buf, size_t bufsize) {
1365 #ifdef ENABLE_OPENSSL
1366  if (client->ssl == NULL) {
1367  return qio_gets(client->socket, buf, bufsize, client->timeoutms);
1368  } else {
1369  if (bufsize <= 1) return -1;
1370 
1371  struct SslConn *ssl = client->ssl;
1372  ssize_t readcnt = 0;
1373  char *ptr;
1374 
1375  for (ptr = buf; readcnt < (bufsize - 1); ptr++) {
1376  // wait readable
1377  //if (qio_wait_readable(client->socket, client->timeoutms) <= 0) {
1378  // break;
1379  //}
1380 
1381  int rsize = SSL_read(ssl->ssl, ptr, 1);
1382  if (rsize != 1) {
1383  unsigned long sslerr = ERR_get_error();
1384  if (sslerr == SSL_ERROR_WANT_READ) {
1385  continue;
1386  }
1387 
1388  DEBUG("OpenSSL: %s (%d)",
1389  ERR_reason_error_string(sslerr), rsize);
1390  break;
1391  }
1392 
1393  readcnt++;
1394  if (*ptr == '\r') ptr--;
1395  else if (*ptr == '\n') break;
1396  }
1397 
1398  *ptr = '\0';
1399  DEBUG("SSL_read: %s (%zd)", buf, readcnt);
1400 
1401  if (readcnt > 0) return readcnt;
1402  return -1;
1403  }
1404 #else
1405  return qio_gets(client->socket, buf, bufsize, client->timeoutms);
1406 #endif
1407 }
1408 
1409 /**
1410  * qhttpclient->read(): Reads data from a HTTP/HTTPS stream.
1411  *
1412  * @param client qhttpclient object pointer.
1413  * @param buf a buffer pointer for storing content. (can be NULL, then
1414  * read & throw out content)
1415  * @param length content size to read.
1416  *
1417  * @return number of bytes readed
1418  *
1419  * @code
1420  * off_t clength = 0;
1421  * int resno = client->readresponse(client, NULL, &clength);
1422  * if(clength > 0) {
1423  * void *buf = malloc(clength);
1424  * client->read(client, buf, clength);
1425  * }
1426  * @endcode
1427  */
1428 static ssize_t read_(qhttpclient_t *client, void *buf, size_t nbytes) {
1429 #ifdef ENABLE_OPENSSL
1430  if (client->ssl == NULL) {
1431  return qio_read(client->socket, buf, nbytes, client->timeoutms);
1432  } else {
1433  if (nbytes == 0) return 0;
1434 
1435  struct SslConn *ssl = client->ssl;
1436  ssize_t total = 0;
1437  while (total < nbytes) {
1438  //if (qio_wait_readable(client->socket, client->timeoutms) <= 0) {
1439  // break;
1440  //}
1441 
1442  int rsize = 0;
1443  if (buf != NULL) {
1444  rsize = SSL_read(ssl->ssl, buf + total, nbytes - total);
1445  } else {
1446  char trash[1024];
1447  int toread = nbytes - total;
1448  if (toread > sizeof(trash)) toread = sizeof(trash);
1449  rsize = SSL_read(ssl->ssl, trash, toread);
1450  }
1451  if (rsize <= 0) {
1452  DEBUG("OpenSSL: %s (%d)",
1453  ERR_reason_error_string(ERR_get_error()), rsize);
1454  unsigned long sslerr = ERR_get_error();
1455  if (sslerr == SSL_ERROR_WANT_READ) {
1456  usleep(1);
1457  continue;
1458  }
1459  break;
1460  }
1461  total += rsize;
1462  }
1463 
1464  DEBUG("SSL_read: %zd", total);
1465  if (total > 0) return total;
1466  return -1;
1467  }
1468 #else
1469  return qio_read(client->socket, buf, nbytes, client->timeoutms);
1470 #endif
1471 }
1472 
1473 /**
1474  * qhttpclient->write(): Writes data to a HTTP/HTTPS stream.
1475  *
1476  * @param client qhttpclient object pointer.
1477  * @param buf a data pointer.
1478  * @param length content size to write.
1479  *
1480  * @return number of bytes written.
1481  */
1482 static ssize_t write_(qhttpclient_t *client, const void *buf, size_t nbytes) {
1483 #ifdef ENABLE_OPENSSL
1484  if (client->ssl == NULL) {
1485  return qio_write(client->socket, buf, nbytes, -1);
1486  } else {
1487  if (nbytes == 0) return 0;
1488 
1489  struct SslConn *ssl = client->ssl;
1490  ssize_t total = 0;
1491  while (total < nbytes) {
1492  errno = 0;
1493  int wsize = SSL_write(ssl->ssl, buf + total, nbytes - total);
1494  if (wsize <= 0) {
1495  DEBUG("OpenSSL: %s (%d)",
1496  ERR_reason_error_string(ERR_get_error()), wsize);
1497  unsigned long sslerr = ERR_get_error();
1498  if (sslerr == SSL_ERROR_WANT_WRITE) {
1499  usleep(1);
1500  continue;
1501  }
1502  break;
1503  }
1504  total += wsize;
1505  }
1506 
1507  DEBUG("SSL_write: %zd/%zu", total, nbytes);
1508  if (total > 0) return total;
1509  return -1;
1510  }
1511 #else
1512  return qio_write(client->socket, buf, nbytes, -1);
1513 #endif
1514 }
1515 
1516 /**
1517  * qhttpclient->recvfile(): Reads data from a HTTP/HTTPS stream and save
1518  * into a file descriptor.
1519  *
1520  * @param client qhttpclient object pointer.
1521  * @param fd output file descriptor
1522  * @param nbytes the number of bytes to read and save.
1523  *
1524  * @return the number of bytes written if successful, otherwise returns -1.
1525  */
1526 static off_t recvfile(qhttpclient_t *client, int fd, off_t nbytes) {
1527  if (nbytes == 0)
1528  return 0;
1529 
1530  unsigned char buf[MAX_ATOMIC_DATA_SIZE];
1531 
1532  off_t total = 0; // total size sent
1533  while (total < nbytes) {
1534  size_t chunksize; // this time sending size
1535  if (nbytes - total <= sizeof(buf))
1536  chunksize = nbytes - total;
1537  else
1538  chunksize = sizeof(buf);
1539 
1540  // read
1541  ssize_t rsize = read_(client, buf, chunksize);
1542  if (rsize <= 0)
1543  break;
1544 
1545  // write
1546  ssize_t wsize = qio_write(fd, buf, rsize, -1);
1547  DEBUG("FILE write: %zd", wsize);
1548  if (wsize <= 0)
1549  break;
1550 
1551  total += wsize;
1552  if (rsize != wsize) {
1553  DEBUG("size mismatch. read:%zd, write:%zd", rsize, wsize);
1554  break;
1555  }
1556  }
1557 
1558  if (total > 0)
1559  return total;
1560  return -1;
1561 }
1562 
1563 /**
1564  * qhttpclient->sendfile(): Send file data to a HTTP/HTTPS stream.
1565  *
1566  * @param client qhttpclient object pointer.
1567  * @param fd input file descriptor
1568  * @param nbytes the number of bytes to read and send.
1569  *
1570  * @return the number of bytes sent if successful, otherwise returns -1.
1571  */
1572 static off_t sendfile_(qhttpclient_t *client, int fd, off_t nbytes) {
1573  if (nbytes == 0)
1574  return 0;
1575 
1576  unsigned char buf[MAX_ATOMIC_DATA_SIZE];
1577 
1578  off_t total = 0; // total size sent
1579  while (total < nbytes) {
1580  size_t chunksize; // this time sending size
1581  if (nbytes - total <= sizeof(buf))
1582  chunksize = nbytes - total;
1583  else
1584  chunksize = sizeof(buf);
1585 
1586  // read
1587  ssize_t rsize = qio_read(fd, buf, chunksize, -1);
1588  DEBUG("FILE read: %zd", rsize);
1589  if (rsize <= 0)
1590  break;
1591 
1592  // write
1593  ssize_t wsize = write_(client, buf, rsize);
1594  if (wsize <= 0)
1595  break;
1596 
1597  total += wsize;
1598  if (rsize != wsize) {
1599  DEBUG("size mismatch. read:%zd, write:%zd", rsize, wsize);
1600  break;
1601  }
1602  }
1603 
1604  if (total > 0)
1605  return total;
1606  return -1;
1607 }
1608 
1609 /**
1610  * qhttpclient->close(): Closes the connection.
1611  *
1612  * @param qhttpclient_t HTTP object pointer
1613  *
1614  * @return true if successful, otherwise returns false
1615  *
1616  * @code
1617  * httpclient->close(httpclient);
1618  * @endcode
1619  */
1620 static bool _close(qhttpclient_t *client) {
1621  if (client->socket < 0)
1622  return false;
1623 
1624 #ifdef ENABLE_OPENSSL
1625  // release ssl connection
1626  if (client->ssl != NULL) {
1627  struct SslConn *ssl = client->ssl;
1628 
1629  if (ssl->ssl != NULL) {
1630  SSL_shutdown(ssl->ssl);
1631  SSL_free(ssl->ssl);
1632  ssl->ssl = NULL;
1633  }
1634 
1635  if (ssl->ctx != NULL) {
1636  SSL_CTX_free(ssl->ctx);
1637  ssl->ctx = NULL;
1638  }
1639  }
1640 #endif
1641 
1642  // shutdown connection
1643  if (client->ssl == NULL && MAX_SHUTDOWN_WAIT >= 0
1644  && shutdown(client->socket, SHUT_WR) == 0) {
1645  char buf[1024];
1646  while (qio_read(client->socket, buf, sizeof(buf), MAX_SHUTDOWN_WAIT) > 0);
1647  }
1648 
1649  // close connection
1650  close(client->socket);
1651  client->socket = -1;
1652  client->connclose = false;
1653 
1654  return true;
1655 }
1656 
1657 /**
1658  * qhttpclient->free(): Free object.
1659  *
1660  * @param qhttpclient_t HTTP object pointer
1661  *
1662  * @note
1663  * If the connection was not closed, it will close the connection first prior
1664  * to de-allocate object.
1665  *
1666  * @code
1667  * httpclient->free(httpclient);
1668  * @endcode
1669  */
1670 static void _free(qhttpclient_t *client) {
1671  if (client->socket >= 0) {
1672  client->close(client);
1673  }
1674 
1675  if (client->ssl != NULL)
1676  free(client->ssl);
1677  if (client->hostname != NULL)
1678  free(client->hostname);
1679  if (client->useragent != NULL)
1680  free(client->useragent);
1681 
1682  free(client);
1683 }
1684 
1685 #ifndef _DOXYGEN_SKIP
1686 static bool _set_socket_option(int socket) {
1687  bool ret = true;
1688 
1689  // linger option
1690  if (SET_TCP_LINGER_TIMEOUT > 0) {
1691  struct linger li;
1692  li.l_onoff = 1;
1693  li.l_linger = SET_TCP_LINGER_TIMEOUT;
1694  if (setsockopt(socket, SOL_SOCKET, SO_LINGER, &li,
1695  sizeof(struct linger)) < 0) {
1696  ret = false;
1697  }
1698  }
1699 
1700  // nodelay option
1701  if (SET_TCP_NODELAY > 0) {
1702  int so_tcpnodelay = 1;
1703  if (setsockopt(socket, IPPROTO_TCP, TCP_NODELAY, &so_tcpnodelay,
1704  sizeof(so_tcpnodelay)) < 0) {
1705  ret = false;
1706  }
1707  }
1708 
1709  return ret;
1710 }
1711 
1712 static bool _parse_uri(const char *uri, bool *protocol, char *hostname,
1713  size_t namesize, int *port) {
1714 
1715  if (!strncasecmp(uri, "http://", CONST_STRLEN("http://"))) {
1716  *protocol = false;
1717  *port = 80;
1718  } else if (!strncasecmp(uri, "https://", CONST_STRLEN("https://"))) {
1719  *protocol = true;
1720  *port = 443;
1721  } else {
1722  return false;
1723  }
1724 
1725  char *t1 = strstr(uri, "://");
1726  t1 += 3;
1727  char *t2 = strstr(t1, "/");
1728  if (t2 == NULL)
1729  t2 = (char *) uri + strlen(uri);
1730 
1731  if (t2 - t1 + 1 > namesize)
1732  return false;
1733  qstrncpy(hostname, namesize, t1, t2 - t1);
1734 
1735  t1 = strstr(hostname, ":");
1736  if (t1 != NULL) {
1737  *t1 = '\0';
1738  *port = atoi(t1 + 1);
1739  }
1740 
1741  return true;
1742 }
1743 #endif /* _DOXYGEN_SKIP */
1744 
1745 #endif /* DISABLE_QHTTPCLIENT */
static bool _close(qhttpclient_t *client)
qhttpclient->close(): Closes the connection.
Definition: qhttpclient.c:1620
static void setkeepalive(qhttpclient_t *client, bool keepalive)
qhttpclient->setkeepalive(): Sets KEEP-ALIVE feature on/off.
Definition: qhttpclient.c:380
static ssize_t gets_(qhttpclient_t *client, char *buf, size_t bufsize)
qhttpclient->gets(): Reads a text line from a HTTP/HTTPS stream.
Definition: qhttpclient.c:1364
static void settimeout(qhttpclient_t *client, int timeoutms)
qhttpclient->settimeout(): Sets connection wait timeout.
Definition: qhttpclient.c:363
ssize_t qio_gets(int fd, char *buf, size_t bufsize, int timeoutms)
Read a line from a file descriptor into the buffer pointed to until either a terminating newline or E...
Definition: qio.c:257
static bool sendrequest(qhttpclient_t *client, const char *method, const char *uri, qlisttbl_t *reqheaders)
qhttpclient->sendrequest(): Sends a HTTP request to the remote host.
Definition: qhttpclient.c:1186
bool qsocket_get_addr(struct sockaddr_in *addr, const char *hostname, int port)
Convert hostname to sockaddr_in structure.
Definition: qsocket.c:134
qlisttbl_t * qlisttbl(int options)
Create a new Q_LIST linked-list container.
Definition: qlisttbl.c:150
char * qstrncpy(char *dst, size_t size, const char *src, size_t nbytes)
Copy src string to dst no more than n bytes.
Definition: qstring.c:339
static off_t sendfile_(qhttpclient_t *client, int fd, off_t nbytes)
qhttpclient->sendfile(): Send file data to a HTTP/HTTPS stream.
Definition: qhttpclient.c:1572
static void * cmd(qhttpclient_t *client, const char *method, const char *uri, void *data, size_t size, int *rescode, size_t *contentslength, qlisttbl_t *reqheaders, qlisttbl_t *resheaders)
qhttpclient->cmd(): Sends a custom request(method) to the remote host and reads it's response...
Definition: qhttpclient.c:1091
static bool put(qhttpclient_t *client, const char *uri, int fd, off_t length, int *rescode, qlisttbl_t *reqheaders, qlisttbl_t *resheaders, bool(*callback)(void *userdata, off_t sentbytes), void *userdata)
qhttpclient->put(): Uploads a file to the remote host using PUT method.
Definition: qhttpclient.c:918
char * qstrtrim(char *str)
Remove white spaces(including CR, LF) from head and tail of the string.
Definition: qstring.c:55
static void setuseragent(qhttpclient_t *client, const char *useragent)
qhttpclient->setuseragent(): Sets user-agent string.
Definition: qhttpclient.c:394
static ssize_t write_(qhttpclient_t *client, const void *buf, size_t nbytes)
qhttpclient->write(): Writes data to a HTTP/HTTPS stream.
Definition: qhttpclient.c:1482
static void _free(qhttpclient_t *client)
qhttpclient->free(): Free object.
Definition: qhttpclient.c:1670
char * qstrcpy(char *dst, size_t size, const char *src)
Copy src string to dst.
Definition: qstring.c:320
static bool setssl(qhttpclient_t *client)
qhttpclient->setssl(): Sets connection to HTTPS connection
Definition: qhttpclient.c:323
static int readresponse(qhttpclient_t *client, qlisttbl_t *resheaders, off_t *contentlength)
qhttpclient->readresponse(): Reads HTTP response header from the remote host.
Definition: qhttpclient.c:1285
ssize_t qio_read(int fd, void *buf, size_t nbytes, int timeoutms)
Read from a file descriptor.
Definition: qio.c:118
qhttpclient_t * qhttpclient(const char *destname, int port)
Initialize & create new HTTP client.
Definition: qhttpclient.c:245
qgrow_t * qgrow(int options)
Initialize grow.
Definition: qgrow.c:134
int qio_wait_writable(int fd, int timeoutms)
Test & wait until the file descriptor is ready for writing.
Definition: qio.c:87
ssize_t qio_write(int fd, const void *buf, size_t nbytes, int timeoutms)
Write to a file descriptor.
Definition: qio.c:159
static off_t recvfile(qhttpclient_t *client, int fd, off_t nbytes)
qhttpclient->recvfile(): Reads data from a HTTP/HTTPS stream and save into a file descriptor...
Definition: qhttpclient.c:1526
static ssize_t read_(qhttpclient_t *client, void *buf, size_t nbytes)
qhttpclient->read(): Reads data from a HTTP/HTTPS stream.
Definition: qhttpclient.c:1428
int qio_wait_readable(int fd, int timeoutms)
Test & wait until the file descriptor has readable data.
Definition: qio.c:59
static bool head(qhttpclient_t *client, const char *uri, int *rescode, qlisttbl_t *reqheaders, qlisttbl_t *resheaders)
qhttpclient->head(): Sends a HEAD request.
Definition: qhttpclient.c:553
static bool open_(qhttpclient_t *client)
qhttpclient->open(): Opens a connection to the remote host.
Definition: qhttpclient.c:417