summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGravatar Christian Hesse <mail@eworm.de>2013-11-07 21:31:25 +0100
committerGravatar Christian Hesse <mail@eworm.de>2013-11-07 21:31:25 +0100
commitf34cf970157155e1e3e6424a171621ab64568864 (patch)
tree0f4e9acef70119c36a1c08452e875ec36c268b55
parent485196be1dd4980af450559468b51122287c65f8 (diff)
downloadpaccache-f34cf970157155e1e3e6424a171621ab64568864.tar.gz
paccache-f34cf970157155e1e3e6424a171621ab64568864.tar.zst
make pacredir multi-threaded, do all curl requests at once
-rw-r--r--Makefile1
-rw-r--r--pacredir.c215
2 files changed, 131 insertions, 85 deletions
diff --git a/Makefile b/Makefile
index d33f02f..b82738f 100644
--- a/Makefile
+++ b/Makefile
@@ -7,6 +7,7 @@ CP := cp
RM := rm
SED := sed
CFLAGS += -O2 -Wall -Werror
+CFLAGS += -lpthread
CFLAGS += $(shell pkg-config --libs --cflags libcurl)
CFLAGS += $(shell pkg-config --libs --cflags avahi-client)
CFLAGS += $(shell pkg-config --libs --cflags libmicrohttpd)
diff --git a/pacredir.c b/pacredir.c
index b448068..549812a 100644
--- a/pacredir.c
+++ b/pacredir.c
@@ -5,6 +5,7 @@
* of the GNU General Public License, incorporated herein by reference.
*/
+/* glibc headers */
#include <arpa/inet.h>
#include <assert.h>
#include <math.h>
@@ -17,14 +18,18 @@
#include <sys/stat.h>
#include <time.h>
+/* Avahi headers */
#include <avahi-client/lookup.h>
#include <avahi-common/error.h>
#include <avahi-common/simple-watch.h>
+/* various headers needing linker options */
#include <curl/curl.h>
#include <iniparser.h>
#include <microhttpd.h>
+#include <pthread.h>
+/* compile time configuration */
#include "config.h"
/* services */
@@ -54,6 +59,22 @@ struct ignore_interfaces {
struct ignore_interfaces * next;
};
+/* request */
+struct request {
+ /* host name */
+ const char * host;
+ /* port */
+ uint16_t port;
+ /* pointer to bad */
+ __time_t * bad;
+ /* url */
+ char * url;
+ /* HTTP status code */
+ long http_code;
+ /* last modified timestamp */
+ long last_modified;
+};
+
/* global variables */
struct hosts * hosts = NULL;
struct ignore_interfaces * ignore_interfaces = NULL;
@@ -98,7 +119,7 @@ int add_host(const char * host, const char * type) {
tmphosts->pacserve.bad = 0;
tmphosts->pacdbserve.online = 0;
tmphosts->pacdbserve.bad = 0;
- tmphosts->next = realloc(tmphosts->next, sizeof(struct hosts));
+ tmphosts->next = malloc(sizeof(struct hosts));
tmphosts->next->host = NULL;
tmphosts->next->next = NULL;
@@ -207,14 +228,16 @@ static void client_callback(AvahiClient *c, AvahiClientState state, AVAHI_GCC_UN
}
/*** get_http_code ***/
-int get_http_code(const char * host, const uint16_t port, const char * url, long * http_code, long * last_modified) {
+static void * get_http_code(void * data) {
+ struct request * request = (struct request *)data;
CURL *curl;
CURLcode res;
+ struct timeval tv;
- curl_global_init(CURL_GLOBAL_ALL);
+ gettimeofday(&tv, NULL);
if ((curl = curl_easy_init()) != NULL) {
- curl_easy_setopt(curl, CURLOPT_URL, url);
+ curl_easy_setopt(curl, CURLOPT_URL, request->url);
/* example.com is redirected, so we tell libcurl to follow redirection */
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
/* set user agent */
@@ -223,41 +246,43 @@ int get_http_code(const char * host, const uint16_t port, const char * url, long
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
/* ask for filetime */
curl_easy_setopt(curl, CURLOPT_FILETIME, 1L);
- /* set connection timeout to 2 seconds
+ /* set connection timeout to 5 seconds
* if the host needs longer we do not want to use it anyway ;) */
- curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 2L);
+ curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 5L);
+ /* time out if connection is established but transfer rate is low
+ * this should make curl finish after a maximum of 8 seconds */
+ curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1L);
+ curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 3L);
/* perform the request */
if (curl_easy_perform(curl) != CURLE_OK) {
- write_log(stderr, "Could not connect to server %s on port %d.\n", host, port);
- *http_code = 0;
- *last_modified = 0;
- return EXIT_FAILURE;
+ write_log(stderr, "Could not connect to server %s on port %d.\n", request->host, request->port);
+ request->http_code = 0;
+ request->last_modified = 0;
+ *request->bad = tv.tv_sec;
+ return NULL;
}
- /* get http code */
- if ((res = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, http_code)) != CURLE_OK) {
+ /* get http status code */
+ if ((res = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &(request->http_code))) != CURLE_OK) {
write_log(stderr, "curl_easy_getinfo() failed: %s\n", curl_easy_strerror(res));
- return EXIT_FAILURE;
+ return NULL;
}
/* get last modified time */
- if (*http_code == MHD_HTTP_OK) {
- if ((res = curl_easy_getinfo(curl, CURLINFO_FILETIME, last_modified)) != CURLE_OK) {
+ if (request->http_code == MHD_HTTP_OK) {
+ if ((res = curl_easy_getinfo(curl, CURLINFO_FILETIME, &(request->last_modified))) != CURLE_OK) {
write_log(stderr, "curl_easy_getinfo() failed: %s\n", curl_easy_strerror(res));
- return EXIT_FAILURE;
+ return NULL;
}
} else
- *last_modified = 0;
+ request->last_modified = 0;
/* always cleanup */
curl_easy_cleanup(curl);
}
- /* we're done with libcurl, so clean it up */
- curl_global_cleanup();
-
- return EXIT_SUCCESS;
+ return NULL;
}
/*** ahc_echo ***
@@ -269,30 +294,38 @@ static int ahc_echo(void * cls, struct MHD_Connection * connection, const char *
int ret;
struct hosts * tmphosts = hosts;
- char * url = NULL, * url_recent = NULL, * page;
+ char * url = NULL, * page;
const char * basename;
struct timeval tv;
struct stat fst;
char * filename;
- unsigned int recent = 0;
- long http_code, last_modified, last_modified_recent = 0;
+ uint8_t dbfile = 0;
+ int i, error, req_count = -1;
+ pthread_t * tid = NULL;
+ struct request ** requests = NULL;
+ struct request * request = NULL;
+ long http_code = 0, last_modified = 0;
/* we want the filename, not the path */
basename = uri;
while (strstr(basename, "/") != NULL)
basename = strstr(basename, "/") + 1;
+ /* unexpected method */
if (strcmp(method, "GET") != 0)
- return MHD_NO; /* unexpected method */
+ return MHD_NO;
+
+ /* The first time only the headers are valid,
+ * do not respond in the first round... */
if (&dummy != *ptr) {
- /* The first time only the headers are valid,
- * do not respond in the first round... */
*ptr = &dummy;
return MHD_YES;
}
+
+ /* upload data in a GET!? */
if (*upload_data_size != 0)
- return MHD_NO; /* upload data in a GET!? */
+ return MHD_NO;
/* clear context pointer */
*ptr = NULL;
@@ -300,77 +333,80 @@ static int ahc_echo(void * cls, struct MHD_Connection * connection, const char *
/* process db file (and signature) request */
if ((strlen(basename) > 3 && strcmp(basename + strlen(basename) - 3, ".db") == 0) ||
(strlen(basename) > 7 && strcmp(basename + strlen(basename) - 7, ".db.sig") == 0)) {
+ dbfile = 1;
/* get timestamp of local file */
filename = malloc(strlen(SYNCPATH) + strlen(basename) + 2);
sprintf(filename, SYNCPATH "/%s", basename);
- bzero(&fst, sizeof(fst));
if (stat(filename, &fst) != 0)
write_log(stderr, "stat() failed, you do not have a local copy of %s\n", basename);
else
- last_modified_recent = fst.st_mtime;
+ last_modified = fst.st_mtime;
free(filename);
+ }
- /* try to find a server with most recent file */
- while (tmphosts->host != NULL) {
- gettimeofday(&tv, NULL);
-
- /* skip host if offline or had a bad request within last BADTIME seconds */
- if (tmphosts->pacdbserve.online == 0 || tmphosts->pacdbserve.bad + BADTIME > tv.tv_sec) {
- tmphosts = tmphosts->next;
- continue;
- }
-
- url = realloc(url, 10 + strlen(tmphosts->host) + (log10(PORT_PACDBSERVE)+1) + strlen(basename));
- sprintf(url, "http://%s:%d/%s", tmphosts->host, PORT_PACDBSERVE, basename);
-
- write_log(stdout, "Trying %s\n", url);
- if (get_http_code(tmphosts->host, PORT_PACDBSERVE, url, &http_code, &last_modified) == EXIT_FAILURE)
- tmphosts->pacdbserve.bad = tv.tv_sec;
- else if (http_code == MHD_HTTP_OK && last_modified > last_modified_recent) {
- if (recent > 0)
- free(url_recent);
- last_modified_recent = last_modified;
- url_recent = url;
- url = NULL;
- recent++;
- }
+ /* try to find a server with most recent file */
+ while (tmphosts->host != NULL) {
+ gettimeofday(&tv, NULL);
+ /* skip host if offline or had a bad request within last BADTIME seconds */
+ if ((dbfile == 1 && (tmphosts->pacdbserve.online == 0 || tmphosts->pacdbserve.bad + BADTIME > tv.tv_sec)) ||
+ (dbfile == 0 && (tmphosts->pacserve.online == 0 || tmphosts->pacserve.bad + BADTIME > tv.tv_sec))) {
tmphosts = tmphosts->next;
+ continue;
}
- if (url != NULL) {
- free(url);
- url = NULL;
+
+ /* This is multi-threading code!
+ * Pointer to struct request does not work as realloc can relocate the data.
+ * We need a pointer to pointer to struct request, store the addresses in
+ * an array and give get_http_code() a struct the does not change! */
+ req_count++;
+ tid = realloc(tid, sizeof(pthread_t) * (req_count + 1));
+ requests = realloc(requests, sizeof(size_t) * (req_count + 1));
+ requests[req_count] = malloc(sizeof(struct request));
+ request = requests[req_count];
+
+ /* prepare request struct */
+ request->host = tmphosts->host;
+ if (dbfile == 1) {
+ request->port = PORT_PACDBSERVE;
+ request->bad = &(tmphosts->pacdbserve.bad);
+ } else {
+ request->port = PORT_PACSERVE;
+ request->bad = &(tmphosts->pacserve.bad);
}
- if (recent > 0) {
- http_code = MHD_HTTP_OK;
- url = url_recent;
- } else
- http_code = 0;
- /* process package file request */
- } else {
- /* try to find a server */
- while (tmphosts->host != NULL) {
- gettimeofday(&tv, NULL);
-
- /* skip host if offline or had a bad request within last BADTIME seconds */
- if (tmphosts->pacserve.online == 0 || tmphosts->pacserve.bad + BADTIME > tv.tv_sec) {
- tmphosts = tmphosts->next;
- continue;
- }
+ request->url = malloc(10 + strlen(tmphosts->host) + 5 /* max strlen of decimal 16bit value */ + strlen(basename));
+ sprintf(request->url, "http://%s:%d/%s", tmphosts->host, dbfile == 1 ? PORT_PACDBSERVE : PORT_PACSERVE, basename);
+ request->http_code = 0;
+ request->last_modified = 0;
- url = realloc(url, 10 + strlen(tmphosts->host) + (log10(PORT_PACSERVE)+1) + strlen(basename));
- sprintf(url, "http://%s:%d/%s", tmphosts->host, PORT_PACSERVE, basename);
+ write_log(stdout, "Trying %s\n", request->url);
- write_log(stdout, "Trying %s\n", url);
- if (get_http_code(tmphosts->host, PORT_PACSERVE, url, &http_code, &last_modified) == EXIT_FAILURE)
- tmphosts->pacserve.bad = tv.tv_sec;
- else if (http_code == MHD_HTTP_OK)
- break;
+ if ((error = pthread_create(&tid[req_count], NULL, get_http_code, (void *)request)) != 0)
+ write_log(stderr, "Could not run thread number %d, errno %d\n", req_count, error);
- tmphosts = tmphosts->next;
- }
+ tmphosts = tmphosts->next;
+ }
+
+ /* try to find a suitable response */
+ for (i = 0; i <= req_count; i++) {
+ if ((error = pthread_join(tid[i], NULL)) != 0)
+ write_log(stderr, "Could not join thread number %d, errno %d\n", i, error);
+
+ request = requests[i];
+
+ if (request->http_code == MHD_HTTP_OK &&
+ (dbfile == 0 || request->last_modified > last_modified)) {
+ printf("Found: %s\n", request->url);
+ if (url != NULL)
+ free(url);
+ url = request->url;
+ http_code = MHD_HTTP_OK;
+ last_modified = request->last_modified;
+ } else
+ free(request->url);
+ free(request);
}
/* give response */
@@ -381,6 +417,7 @@ static int ahc_echo(void * cls, struct MHD_Connection * connection, const char *
response = MHD_create_response_from_data(strlen(page), (void*) page, MHD_NO, MHD_NO);
ret = MHD_add_response_header(response, "Location", url);
ret = MHD_queue_response(connection, MHD_HTTP_TEMPORARY_REDIRECT, response);
+ free(url);
} else {
write_log(stdout, "File %s not found, giving up.\n", basename);
page = malloc(strlen(PAGE404) + strlen(basename) + 1);
@@ -391,8 +428,10 @@ static int ahc_echo(void * cls, struct MHD_Connection * connection, const char *
MHD_destroy_response(response);
free(page);
- if (url != NULL)
- free(url);
+ if (req_count > -1) {
+ free(tid);
+ free(requests);
+ }
return ret;
}
@@ -445,7 +484,6 @@ int main(int argc, char ** argv) {
ignore_interfaces->interface = NULL;
ignore_interfaces->next = NULL;
-
/* parse config file */
if ((ini = iniparser_load(CONFIGFILE)) == NULL) {
write_log(stderr, "cannot parse file " CONFIGFILE ", continue anyway\n");
@@ -532,6 +570,9 @@ int main(int argc, char ** argv) {
return EXIT_FAILURE;
}
+ /* initialize curl */
+ curl_global_init(CURL_GLOBAL_ALL);
+
/* register signal callbacks */
signal(SIGTERM, sig_callback);
signal(SIGINT, sig_callback);
@@ -540,8 +581,12 @@ int main(int argc, char ** argv) {
/* run the main loop */
avahi_simple_poll_loop(simple_poll);
+ /* stop http server */
MHD_stop_daemon(mhd);
+ /* we're done with libcurl, so clean it up */
+ curl_global_cleanup();
+
ret = EXIT_SUCCESS;
fail: