commit decce743497b6eac8af5ad9e13441ec76e77b162
parent 341f4d21eecb62aa246216fea138a2ea59f7b8e1
Author: citbl <citbl@citbl.org>
Date: Tue, 28 Oct 2025 20:57:09 +1000
1.6 multithreaded fetch of feeds 🚀
Diffstat:
9 files changed, 156 insertions(+), 43 deletions(-)
diff --git a/docs/readr.1 b/docs/readr.1
@@ -5,7 +5,7 @@
.nh
.ad l
.\" Begin generated content:
-.TH "READR" "1" "2025-10-25"
+.TH "READR" "1" "2025-10-28"
.PP
.SH NAME
.PP
diff --git a/docs/readr.html b/docs/readr.html
@@ -88,7 +88,7 @@ https://example3.com/rss</pre>
</div>
<table class="foot">
<tr>
- <td class="foot-date">2025-10-25</td>
+ <td class="foot-date">2025-10-28</td>
<td class="foot-os"></td>
</tr>
</table>
diff --git a/makefile b/makefile
@@ -1,5 +1,5 @@
APP := readr
-PKG := $(shell pkg-config --cflags --libs mrss sqlite3)
+PKG := $(shell pkg-config --cflags --libs mrss sqlite3 libcurl)
CFLAGS := -std=c23 -g -Wall -Wextra -fsanitize=address -fsanitize=undefined
CREL := -std=c23 -Wall -Wextra -Wpedantic -Werror -O3
diff --git a/src/config.h b/src/config.h
@@ -1,6 +1,6 @@
#pragma once
-#define VERSION "v1.5"
+#define VERSION "v1.6"
#define USER_AGENT \
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 " \
@@ -11,6 +11,10 @@
// folder and file must exist
#define FEEDS "~/.config/readr/feeds"
+// limited by the theorical limit of your OS "threads per process"
+// 2048 (safe), up to 8196 if configured on modern macOS
+#define MAX_FEEDS 512
+
// folder must exist, file will be automatically created
// can technically be cloud shared, dropbox etc. for replication
#define DB_PATH "~/.config/readr/posts.sqlite"
diff --git a/src/feeds.c b/src/feeds.c
@@ -4,67 +4,84 @@
#include <mrss.h>
#include <stdlib.h>
#include <string.h>
+#include <pthread.h>
+#include <curl/curl.h>
#include "config.h"
#include "utils.h"
-#include "db.h"
+#include "http.h"
-void fetch_feed(feed_t*, char*);
+static void parse_feed(feed_t*, char*, char*);
static void populate_feed(feed_t*);
app_t
-load_app(char* contents)
+load_app(char* contents_feeds)
{
+ CURLcode res = curl_global_init(CURL_GLOBAL_ALL);
+ if (res) panic("Curl init error %d", res);
+ pthread_t t[MAX_FEEDS];
+ char* lines[MAX_FEEDS];
+ int feed_count = 0;
+
app_t app = {
.feeds_cap = FEEDS_CAP,
- .feeds = ecalloc(FEEDS_CAP, sizeof(char*)),
+ .feeds = ecalloc(FEEDS_CAP, sizeof(feed_t*)),
};
- remove_all_chars(contents, '\r');
- char* line = strtok(contents, "\n");
+ remove_all_chars(contents_feeds, '\r');
+ char* line = strtok(contents_feeds, "\n");
while (line != NULL) {
- feed_t* feed = (feed_t*)ecalloc(1, sizeof(feed_t));
+ lines[feed_count] = strdup(line);
+
+ /* threads out --> */
+ pthread_create(&t[feed_count], NULL, threaded_fetch, lines[feed_count]);
+ /* threads out --> */
- fetch_feed(feed, line); // gets the content and loads it into db
- printf(".");
- fflush(stdout);
+ feed_count++;
+ if (feed_count > MAX_FEEDS) {
+ fprintf(stderr, "WARNING exeeded number of feeds %d\n", MAX_FEEDS);
+ break;
+ }
+ line = strtok(NULL, "\n");
+ }
- populate_feed(feed); // fetch back from db top 100
+ for (int i = 0; i < feed_count; i++) {
+ void* ret;
+
+ /* <-- thread join */
+ pthread_join(t[i], &ret);
+ /* <-- thread join */
+
+ http_blob* httpblob = (http_blob*)ret;
+ feed_t* feed = (feed_t*)ecalloc(1, sizeof(feed_t));
+ parse_feed(feed, lines[i], httpblob->data);
+ populate_feed(feed);
+ free(lines[i]);
+ http_blob_free(ret);
if (app.feeds_cap == app.feeds_len) {
app.feeds_cap *= 2;
- app.feeds = realloc(app.feeds, (size_t)app.feeds_cap);
+ app.feeds = realloc(app.feeds, app.feeds_cap * sizeof(feed_t*));
}
app.feeds[app.feeds_len++] = feed;
- line = strtok(NULL, "\n");
}
+ free(line);
+ curl_global_cleanup();
return app;
}
-void
-fetch_feed(feed_t* feed, char* url)
+static void
+parse_feed(feed_t* feed, char* url, char* http_body)
{
- static mrss_t* rss = NULL;
-
- mrss_options_t* opt;
- opt = mrss_options_new(TIMEOUT, // timeout
- NULL, // proxy
- NULL, // proxy_auth
- NULL, // certfile
- NULL, // password
- NULL, // cacert
- 1, // verifypeer
- NULL, // authentication
- USER_AGENT // for reddit etc. using Chrome
- );
+ mrss_t* rss = NULL;
*feed = (feed_t) { .url = url };
feed->title = (char*)ecalloc(FEED_CAP, sizeof(char));
- /* process feed */
- mrss_error_t rc = mrss_parse_url_with_options(url, &rss, opt);
+ /* process feed from given buffer */
+ mrss_error_t rc = mrss_parse_buffer(http_body, strlen(http_body), &rss);
if (rc != MRSS_OK || rss == NULL) {
snprintf(feed->title, FEED_CAP, "%s%s ", "(bad) ", url ? url : "(unknown feed url)");
@@ -95,6 +112,8 @@ fetch_feed(feed_t* feed, char* url)
}
mrss_free(rss);
+ printf(".");
+ fflush(stdout);
}
static void
diff --git a/src/http.c b/src/http.c
@@ -0,0 +1,76 @@
+#include <string.h>
+#include <mrss.h>
+#include <string.h>
+#include <pthread.h>
+
+#include "http.h"
+#include "utils.h"
+#include "config.h"
+
+size_t
+http_write(char* ptr, size_t size, size_t nmemb, void* userdata)
+{
+ size_t n = size * nmemb;
+ http_blob* b = userdata;
+ char* p = realloc(b->data, b->len + n + 1);
+ if (!p) return 0;
+ b->data = p;
+ memcpy(b->data + b->len, ptr, n);
+ b->len += n;
+ b->data[b->len] = '\0';
+ return n;
+}
+
+void* /* http contents */
+threaded_fetch(void* url)
+{
+ char curl_errbuf[CURL_ERROR_SIZE];
+ CURL* curl = curl_easy_init();
+ if (!curl) { return NULL; }
+
+ http_blob* blob = ecalloc(1, sizeof *blob);
+ if (!blob) {
+ curl_easy_cleanup(curl);
+ return NULL;
+ }
+
+ curl_easy_setopt(curl, CURLOPT_URL, (char*)url);
+ curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errbuf);
+ curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+ curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L);
+
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, http_write);
+ curl_easy_setopt(curl, CURLOPT_WRITEDATA, blob);
+ curl_easy_setopt(curl, CURLOPT_USERAGENT, USER_AGENT);
+ curl_easy_setopt(curl, CURLOPT_TIMEOUT, TIMEOUT);
+
+ CURLcode res = curl_easy_perform(curl);
+ if (res != CURLE_OK) {
+ fprintf(stderr,
+ "curl failed for %s: %s\n",
+ (char*)url,
+ curl_errbuf[0] ? curl_errbuf : curl_easy_strerror(res));
+ free(blob->data);
+ free(blob);
+ curl_easy_cleanup(curl);
+ return NULL;
+ }
+
+ curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &blob->status);
+ char* ct = NULL;
+ if (curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &ct) == CURLE_OK && ct) {
+ blob->ctype = strdup(ct);
+ }
+
+ curl_easy_cleanup(curl);
+ return blob;
+}
+
+void
+http_blob_free(http_blob* b)
+{
+ if (!b) return;
+ free(b->data);
+ free(b->ctype);
+ free(b);
+}
diff --git a/src/http.h b/src/http.h
@@ -0,0 +1,14 @@
+#pragma once
+
+#include <stdlib.h>
+
+typedef struct {
+ char* data;
+ size_t len;
+ long status;
+ char* ctype;
+} http_blob;
+
+size_t http_write(char* ptr, size_t size, size_t nmemb, void* userdata);
+void* threaded_fetch(void* url);
+void http_blob_free(http_blob* b);
diff --git a/src/readr.h b/src/readr.h
@@ -27,8 +27,8 @@ typedef struct {
typedef struct {
feed_t** feeds;
- int feeds_len, feeds_cap;
+ size_t feeds_len, feeds_cap;
+ size_t selected_feed;
int selected_panel;
- int selected_feed;
int selected_post;
} app_t;
diff --git a/src/render.c b/src/render.c
@@ -58,7 +58,7 @@ draw_background(void)
void
render(app_t* app)
{
- int i;
+ size_t i;
uintattr_t color;
char title[TITLE_CAP] = { 0 };
char domain[DOMAIN_CAP] = { 0 };
@@ -81,15 +81,15 @@ render(app_t* app)
else
color = (FEED_COLOR | TB_DIM);
- tb_print(2, 3 + i, color, BACK_COLOR, feed->title ? feed->title : "N/A");
+ tb_print(2, 3 + (int)i, color, BACK_COLOR, feed->title ? feed->title : "N/A");
}
// posts
int visible_len = MIN(buffer_height, selected_feed->posts_len);
- for (i = 0; i < visible_len; i++) {
- post = selected_feed->posts[i];
+ for (int j = 0; j < visible_len; j++) {
+ post = selected_feed->posts[j];
// title - added space for when title and domain overlap
snprintf(title, TITLE_CAP, "%s ", post->title ? post->title : "(bad title");
@@ -99,7 +99,7 @@ render(app_t* app)
snprintf(domain, DOMAIN_CAP, "%s", domain_host ? domain_host : "(bad title)");
int domain_len = (int)strlen(domain);
- if (app->selected_panel == 1 && i == app->selected_post)
+ if (app->selected_panel == 1 && j == app->selected_post)
if (post->seen)
color = (POST_COLOR | TB_REVERSE | TB_DIM);
else
@@ -109,8 +109,8 @@ render(app_t* app)
else
color = (POST_COLOR);
- tb_print(width - domain_len - 2, 3 + i, DOMAIN_COLOR, BACK_COLOR, domain);
- tb_print(FEED_CAP + 4, 3 + i, color, BACK_COLOR, title);
+ tb_print(width - domain_len - 2, 3 + (int)j, DOMAIN_COLOR, BACK_COLOR, domain);
+ tb_print(FEED_CAP + 4, 3 + (int)j, color, BACK_COLOR, title);
}
tb_present();
