mirror of
https://git.bwaaa.monster/omnisearch
synced 2026-03-25 17:19:02 +02:00
Compare commits
9 Commits
8229aaa489
...
indev
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
783a58d954 | ||
|
|
e9b01902d9 | ||
|
|
8b7b8de06c | ||
|
|
5a4af40b74 | ||
|
|
4ed9ec9fc5 | ||
|
|
660a4918b8 | ||
|
|
51e7fcaad2 | ||
|
|
bcee71cbbb | ||
|
|
6b90877869 |
@@ -25,3 +25,9 @@ domain = https://search.example.com
|
||||
|
||||
# Cache TTL for infobox data in seconds (default: 86400 = 24 hours)
|
||||
#ttl_infobox = 86400
|
||||
|
||||
[engines]
|
||||
# Use * for all engines, or specify comma-separated list (e.g., ddg,yahoo)
|
||||
# Use *,-engine to exclude specific engines (e.g., *,-startpage)
|
||||
# Available engines: ddg, startpage, yahoo, mojeek
|
||||
engines="*"
|
||||
|
||||
136
module.nix
Normal file
136
module.nix
Normal file
@@ -0,0 +1,136 @@
|
||||
self:
|
||||
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
cfg = config.services.omnisearch;
|
||||
pkg = cfg.package;
|
||||
|
||||
finalConfigFile =
|
||||
if cfg.configFile != null then
|
||||
cfg.configFile
|
||||
else
|
||||
pkgs.writeText "omnisearch.ini" ''
|
||||
[server]
|
||||
host = ${cfg.settings.server.host}
|
||||
port = ${toString cfg.settings.server.port}
|
||||
domain = ${cfg.settings.server.domain}
|
||||
|
||||
[proxy]
|
||||
${lib.optionalString (cfg.settings.proxy.proxy != null) "proxy = \"${cfg.settings.proxy.proxy}\""}
|
||||
${lib.optionalString (
|
||||
cfg.settings.proxy.list_file != null
|
||||
) "list_file = ${cfg.settings.proxy.list_file}"}
|
||||
max_retries = ${toString cfg.settings.proxy.max_retries}
|
||||
randomize_username = ${lib.boolToString cfg.settings.proxy.randomize_username}
|
||||
randomize_password = ${lib.boolToString cfg.settings.proxy.randomize_password}
|
||||
|
||||
[cache]
|
||||
dir = ${cfg.settings.cache.dir}
|
||||
ttl_search = ${toString cfg.settings.cache.ttl_search}
|
||||
ttl_infobox = ${toString cfg.settings.cache.ttl_infobox}
|
||||
'';
|
||||
in
|
||||
{
|
||||
options.services.omnisearch = {
|
||||
enable = lib.mkEnableOption "OmniSearch metasearch engine";
|
||||
|
||||
package = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
default = self.packages.${pkgs.stdenv.hostPlatform.system}.default;
|
||||
description = "The omnisearch package to use.";
|
||||
};
|
||||
|
||||
configFile = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.path;
|
||||
default = null;
|
||||
description = "Path to a custom config.ini. Overrides 'settings'.";
|
||||
};
|
||||
|
||||
settings = {
|
||||
server = {
|
||||
host = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "0.0.0.0";
|
||||
};
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 8087;
|
||||
};
|
||||
domain = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "http://localhost:8087";
|
||||
};
|
||||
};
|
||||
proxy = {
|
||||
proxy = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
};
|
||||
list_file = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
};
|
||||
max_retries = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
default = 3;
|
||||
};
|
||||
randomize_username = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
};
|
||||
randomize_password = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
};
|
||||
};
|
||||
cache = {
|
||||
dir = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/var/cache/omnisearch";
|
||||
};
|
||||
ttl_search = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
default = 3600;
|
||||
};
|
||||
ttl_infobox = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
default = 86400;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
systemd.services.omnisearch = {
|
||||
description = "OmniSearch Service";
|
||||
after = [ "network.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
||||
serviceConfig = {
|
||||
ExecStart = "${pkg}/bin/omnisearch";
|
||||
|
||||
WorkingDirectory = "/var/lib/omnisearch";
|
||||
StateDirectory = "omnisearch";
|
||||
CacheDirectory = "omnisearch";
|
||||
|
||||
BindReadOnlyPaths = [
|
||||
"${pkg}/share/omnisearch/templates:/var/lib/omnisearch/templates"
|
||||
"${pkg}/share/omnisearch/static:/var/lib/omnisearch/static"
|
||||
"${finalConfigFile}:/var/lib/omnisearch/config.ini"
|
||||
];
|
||||
|
||||
DynamicUser = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
PrivateTmp = true;
|
||||
Restart = "always";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -54,6 +54,8 @@ int load_config(const char *filename, Config *config) {
|
||||
value_end--;
|
||||
}
|
||||
|
||||
while (*value == ' ' || *value == '\t')
|
||||
value++;
|
||||
while (*value == '"' || *value == '\'')
|
||||
value++;
|
||||
|
||||
@@ -91,6 +93,11 @@ int load_config(const char *filename, Config *config) {
|
||||
} else if (strcmp(key, "ttl_infobox") == 0) {
|
||||
config->cache_ttl_infobox = atoi(value);
|
||||
}
|
||||
} else if (strcmp(section, "engines") == 0) {
|
||||
if (strcmp(key, "engines") == 0) {
|
||||
strncpy(config->engines, value, sizeof(config->engines) - 1);
|
||||
config->engines[sizeof(config->engines) - 1] = '\0';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
#define MD5_HASH_LEN 32
|
||||
#define HEX_CHARS "0123456789abcdef"
|
||||
|
||||
#define INFOBOX_FIELD_COUNT 4
|
||||
#define INFOBOX_FIELD_COUNT 5
|
||||
#define MAX_RESULTS_PER_ENGINE 10
|
||||
|
||||
#define CURL_TIMEOUT_SECS 15L
|
||||
@@ -42,6 +42,7 @@ typedef struct {
|
||||
char cache_dir[512];
|
||||
int cache_ttl_search;
|
||||
int cache_ttl_infobox;
|
||||
char engines[512];
|
||||
} Config;
|
||||
|
||||
int load_config(const char *filename, Config *config);
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
#include "Scraping/Scraping.h"
|
||||
|
||||
Config global_config;
|
||||
|
||||
|
||||
int handle_opensearch(UrlParams *params) {
|
||||
(void)params;
|
||||
extern Config global_config;
|
||||
@@ -51,7 +51,8 @@ int main() {
|
||||
.randomize_password = 0,
|
||||
.cache_dir = DEFAULT_CACHE_DIR,
|
||||
.cache_ttl_search = DEFAULT_CACHE_TTL_SEARCH,
|
||||
.cache_ttl_infobox = DEFAULT_CACHE_TTL_INFOBOX};
|
||||
.cache_ttl_infobox = DEFAULT_CACHE_TTL_INFOBOX,
|
||||
.engines = ""};
|
||||
|
||||
if (load_config("config.ini", &cfg) != 0) {
|
||||
fprintf(stderr, "[WARN] Could not load config file, using defaults\n");
|
||||
@@ -59,6 +60,8 @@ int main() {
|
||||
|
||||
global_config = cfg;
|
||||
|
||||
apply_engines_config(cfg.engines);
|
||||
|
||||
if (cache_init(cfg.cache_dir) != 0) {
|
||||
fprintf(stderr,
|
||||
"[WARN] Failed to initialize cache, continuing without caching\n");
|
||||
|
||||
@@ -27,6 +27,12 @@ typedef struct {
|
||||
char *(*url_construct_fn)(const char *query);
|
||||
} InfoBoxHandler;
|
||||
|
||||
enum {
|
||||
RESULT_FIELD_COUNT = 6,
|
||||
LINK_FIELD_COUNT = 3,
|
||||
PAGER_WINDOW_SIZE = 5,
|
||||
};
|
||||
|
||||
static InfoBox fetch_wiki_wrapper(char *query) {
|
||||
char *url = construct_wiki_url(query);
|
||||
if (!url)
|
||||
@@ -53,7 +59,31 @@ static InfoBox fetch_unit_wrapper(char *query) {
|
||||
static InfoBox fetch_currency_wrapper(char *query) {
|
||||
return fetch_currency_data(query);
|
||||
}
|
||||
char *get_base_url(const char *input) {
|
||||
if (!input) return NULL;
|
||||
|
||||
const char *start = input;
|
||||
|
||||
const char *protocol_pos = strstr(input, "://");
|
||||
if (protocol_pos) {
|
||||
start = protocol_pos + 3;
|
||||
}
|
||||
|
||||
const char *end = start;
|
||||
while (*end && *end != '/' && *end != '?' && *end != '#') {
|
||||
end++;
|
||||
}
|
||||
|
||||
size_t len = end - start;
|
||||
|
||||
char *domain = (char *)malloc(len + 1);
|
||||
if (!domain) return NULL;
|
||||
|
||||
strncpy(domain, start, len);
|
||||
domain[len] = '\0';
|
||||
|
||||
return domain;
|
||||
}
|
||||
static int is_calculator_query(const char *query) {
|
||||
if (!query)
|
||||
return 0;
|
||||
@@ -150,11 +180,72 @@ static int add_infobox_to_collection(InfoBox *infobox, char ****collection,
|
||||
(*collection)[current_count][2] =
|
||||
infobox->extract ? strdup(infobox->extract) : NULL;
|
||||
(*collection)[current_count][3] = infobox->url ? strdup(infobox->url) : NULL;
|
||||
(*collection)[current_count][4] = infobox->url ? strdup(infobox->url) : NULL;
|
||||
(*inner_counts)[current_count] = INFOBOX_FIELD_COUNT;
|
||||
|
||||
return current_count + 1;
|
||||
}
|
||||
|
||||
static int add_link_to_collection(const char *href, const char *label,
|
||||
const char *class_name, char ****collection,
|
||||
int **inner_counts, int current_count) {
|
||||
char ***old_collection = *collection;
|
||||
int *old_inner_counts = *inner_counts;
|
||||
char ***new_collection =
|
||||
(char ***)malloc(sizeof(char **) * (current_count + 1));
|
||||
int *new_inner_counts =
|
||||
(int *)malloc(sizeof(int) * (current_count + 1));
|
||||
|
||||
if (!new_collection || !new_inner_counts) {
|
||||
free(new_collection);
|
||||
free(new_inner_counts);
|
||||
return current_count;
|
||||
}
|
||||
|
||||
if (*collection && current_count > 0) {
|
||||
memcpy(new_collection, *collection, sizeof(char **) * current_count);
|
||||
}
|
||||
if (*inner_counts && current_count > 0) {
|
||||
memcpy(new_inner_counts, *inner_counts, sizeof(int) * current_count);
|
||||
}
|
||||
|
||||
*collection = new_collection;
|
||||
*inner_counts = new_inner_counts;
|
||||
|
||||
(*collection)[current_count] =
|
||||
(char **)malloc(sizeof(char *) * LINK_FIELD_COUNT);
|
||||
if (!(*collection)[current_count]) {
|
||||
*collection = old_collection;
|
||||
*inner_counts = old_inner_counts;
|
||||
free(new_collection);
|
||||
free(new_inner_counts);
|
||||
return current_count;
|
||||
}
|
||||
|
||||
(*collection)[current_count][0] = strdup(href ? href : "");
|
||||
(*collection)[current_count][1] = strdup(label ? label : "");
|
||||
(*collection)[current_count][2] = strdup(class_name ? class_name : "");
|
||||
|
||||
if (!(*collection)[current_count][0] || !(*collection)[current_count][1] ||
|
||||
!(*collection)[current_count][2]) {
|
||||
free((*collection)[current_count][0]);
|
||||
free((*collection)[current_count][1]);
|
||||
free((*collection)[current_count][2]);
|
||||
free((*collection)[current_count]);
|
||||
*collection = old_collection;
|
||||
*inner_counts = old_inner_counts;
|
||||
free(new_collection);
|
||||
free(new_inner_counts);
|
||||
return current_count;
|
||||
}
|
||||
|
||||
(*inner_counts)[current_count] = LINK_FIELD_COUNT;
|
||||
|
||||
free(old_collection);
|
||||
free(old_inner_counts);
|
||||
return current_count + 1;
|
||||
}
|
||||
|
||||
static int add_warning_to_collection(const char *engine_name,
|
||||
const char *warning_message,
|
||||
char ****collection, int **inner_counts,
|
||||
@@ -216,9 +307,80 @@ static const char *warning_message_for_job(const ScrapeJob *job) {
|
||||
}
|
||||
}
|
||||
|
||||
static int engine_id_matches(const char *left, const char *right) {
|
||||
if (!left || !right)
|
||||
return 0;
|
||||
|
||||
while (*left && *right) {
|
||||
char l = *left;
|
||||
char r = *right;
|
||||
|
||||
if (l >= 'A' && l <= 'Z')
|
||||
l = l - 'A' + 'a';
|
||||
if (r >= 'A' && r <= 'Z')
|
||||
r = r - 'A' + 'a';
|
||||
|
||||
if (l != r)
|
||||
return 0;
|
||||
|
||||
left++;
|
||||
right++;
|
||||
}
|
||||
|
||||
return *left == *right;
|
||||
}
|
||||
|
||||
static const SearchEngine *find_enabled_engine(const char *engine_id) {
|
||||
if (!engine_id || engine_id[0] == '\0' || engine_id_matches(engine_id, "all"))
|
||||
return NULL;
|
||||
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
if (ENGINE_REGISTRY[i].enabled &&
|
||||
engine_id_matches(ENGINE_REGISTRY[i].id, engine_id)) {
|
||||
return &ENGINE_REGISTRY[i];
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static char *build_search_href(const char *query, const char *engine_id,
|
||||
int page) {
|
||||
const char *safe_query = query ? query : "";
|
||||
int use_engine = engine_id && engine_id[0] != '\0' &&
|
||||
!engine_id_matches(engine_id, "all");
|
||||
size_t needed = strlen("/search?q=") + strlen(safe_query) + 1;
|
||||
|
||||
if (use_engine)
|
||||
needed += strlen("&engine=") + strlen(engine_id);
|
||||
if (page > 1)
|
||||
needed += strlen("&p=") + 16;
|
||||
|
||||
char *href = (char *)malloc(needed);
|
||||
if (!href)
|
||||
return NULL;
|
||||
|
||||
snprintf(href, needed, "/search?q=%s", safe_query);
|
||||
|
||||
if (use_engine) {
|
||||
strcat(href, "&engine=");
|
||||
strcat(href, engine_id);
|
||||
}
|
||||
|
||||
if (page > 1) {
|
||||
char page_buf[16];
|
||||
snprintf(page_buf, sizeof(page_buf), "%d", page);
|
||||
strcat(href, "&p=");
|
||||
strcat(href, page_buf);
|
||||
}
|
||||
|
||||
return href;
|
||||
}
|
||||
|
||||
int results_handler(UrlParams *params) {
|
||||
TemplateContext ctx = new_context();
|
||||
char *raw_query = "";
|
||||
const char *selected_engine_id = "all";
|
||||
int page = 1;
|
||||
int btnI = 0;
|
||||
|
||||
@@ -230,6 +392,8 @@ int results_handler(UrlParams *params) {
|
||||
int parsed = atoi(params->params[i].value);
|
||||
if (parsed > 1)
|
||||
page = parsed;
|
||||
} else if (strcmp(params->params[i].key, "engine") == 0) {
|
||||
selected_engine_id = params->params[i].value;
|
||||
} else if (strcmp(params->params[i].key, "btnI") == 0) {
|
||||
btnI = atoi(params->params[i].value);
|
||||
}
|
||||
@@ -237,19 +401,9 @@ int results_handler(UrlParams *params) {
|
||||
}
|
||||
|
||||
context_set(&ctx, "query", raw_query);
|
||||
|
||||
char page_str[16], prev_str[16], next_str[16], two_prev_str[16],
|
||||
two_next_str[16];
|
||||
char page_str[16];
|
||||
snprintf(page_str, sizeof(page_str), "%d", page);
|
||||
snprintf(prev_str, sizeof(prev_str), "%d", page > 1 ? page - 1 : 0);
|
||||
snprintf(next_str, sizeof(next_str), "%d", page + 1);
|
||||
snprintf(two_prev_str, sizeof(two_prev_str), "%d", page > 2 ? page - 2 : 0);
|
||||
snprintf(two_next_str, sizeof(two_next_str), "%d", page + 2);
|
||||
context_set(&ctx, "page", page_str);
|
||||
context_set(&ctx, "prev_page", prev_str);
|
||||
context_set(&ctx, "next_page", next_str);
|
||||
context_set(&ctx, "two_prev_page", two_prev_str);
|
||||
context_set(&ctx, "two_next_page", two_next_str);
|
||||
|
||||
if (!raw_query || strlen(raw_query) == 0) {
|
||||
send_response("<h1>No query provided</h1>");
|
||||
@@ -257,6 +411,23 @@ int results_handler(UrlParams *params) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
const SearchEngine *selected_engine = find_enabled_engine(selected_engine_id);
|
||||
if (!selected_engine)
|
||||
selected_engine_id = "all";
|
||||
|
||||
context_set(&ctx, "selected_engine", selected_engine_id);
|
||||
char *search_href = build_search_href(raw_query, selected_engine_id, 1);
|
||||
context_set(&ctx, "search_href", search_href ? search_href : "/search");
|
||||
free(search_href);
|
||||
|
||||
int enabled_engine_count = 0;
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
if (ENGINE_REGISTRY[i].enabled &&
|
||||
(!selected_engine || &ENGINE_REGISTRY[i] == selected_engine)) {
|
||||
enabled_engine_count++;
|
||||
}
|
||||
}
|
||||
|
||||
pthread_t infobox_threads[HANDLER_COUNT];
|
||||
InfoBoxThreadData infobox_data[HANDLER_COUNT];
|
||||
|
||||
@@ -276,23 +447,78 @@ int results_handler(UrlParams *params) {
|
||||
ScrapeJob jobs[ENGINE_COUNT];
|
||||
SearchResult *all_results[ENGINE_COUNT];
|
||||
|
||||
int engine_idx = 0;
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
all_results[i] = NULL;
|
||||
jobs[i].engine = &ENGINE_REGISTRY[i];
|
||||
jobs[i].query = raw_query;
|
||||
jobs[i].out_results = &all_results[i];
|
||||
jobs[i].max_results = MAX_RESULTS_PER_ENGINE;
|
||||
jobs[i].results_count = 0;
|
||||
jobs[i].page = page;
|
||||
jobs[i].handle = NULL;
|
||||
jobs[i].response.memory = NULL;
|
||||
jobs[i].response.size = 0;
|
||||
jobs[i].response.capacity = 0;
|
||||
jobs[i].http_status = 0;
|
||||
jobs[i].status = SCRAPE_STATUS_PENDING;
|
||||
if (ENGINE_REGISTRY[i].enabled &&
|
||||
(!selected_engine || &ENGINE_REGISTRY[i] == selected_engine)) {
|
||||
all_results[engine_idx] = NULL;
|
||||
jobs[engine_idx].engine = &ENGINE_REGISTRY[i];
|
||||
jobs[engine_idx].query = raw_query;
|
||||
jobs[engine_idx].out_results = &all_results[engine_idx];
|
||||
jobs[engine_idx].max_results = MAX_RESULTS_PER_ENGINE;
|
||||
jobs[engine_idx].results_count = 0;
|
||||
jobs[engine_idx].page = page;
|
||||
jobs[engine_idx].handle = NULL;
|
||||
jobs[engine_idx].response.memory = NULL;
|
||||
jobs[engine_idx].response.size = 0;
|
||||
jobs[engine_idx].response.capacity = 0;
|
||||
jobs[engine_idx].http_status = 0;
|
||||
jobs[engine_idx].status = SCRAPE_STATUS_PENDING;
|
||||
engine_idx++;
|
||||
}
|
||||
}
|
||||
|
||||
scrape_engines_parallel(jobs, ENGINE_COUNT);
|
||||
int filter_engine_count = 0;
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
if (ENGINE_REGISTRY[i].enabled)
|
||||
filter_engine_count++;
|
||||
}
|
||||
|
||||
if (filter_engine_count > 1) {
|
||||
char ***filter_matrix = NULL;
|
||||
int *filter_inner_counts = NULL;
|
||||
int filter_count = 0;
|
||||
char *all_href = build_search_href(raw_query, "all", 1);
|
||||
|
||||
filter_count = add_link_to_collection(
|
||||
all_href, "All",
|
||||
selected_engine ? "engine-filter" : "engine-filter active",
|
||||
&filter_matrix, &filter_inner_counts, filter_count);
|
||||
free(all_href);
|
||||
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
if (!ENGINE_REGISTRY[i].enabled)
|
||||
continue;
|
||||
|
||||
char *filter_href =
|
||||
build_search_href(raw_query, ENGINE_REGISTRY[i].id, 1);
|
||||
const char *filter_class =
|
||||
(selected_engine && &ENGINE_REGISTRY[i] == selected_engine)
|
||||
? "engine-filter active"
|
||||
: "engine-filter";
|
||||
|
||||
filter_count = add_link_to_collection(filter_href, ENGINE_REGISTRY[i].name,
|
||||
filter_class, &filter_matrix,
|
||||
&filter_inner_counts, filter_count);
|
||||
free(filter_href);
|
||||
}
|
||||
|
||||
if (filter_count > 0) {
|
||||
context_set_array_of_arrays(&ctx, "engine_filters", filter_matrix,
|
||||
filter_count, filter_inner_counts);
|
||||
for (int i = 0; i < filter_count; i++) {
|
||||
for (int j = 0; j < LINK_FIELD_COUNT; j++)
|
||||
free(filter_matrix[i][j]);
|
||||
free(filter_matrix[i]);
|
||||
}
|
||||
free(filter_matrix);
|
||||
free(filter_inner_counts);
|
||||
}
|
||||
}
|
||||
|
||||
if (engine_idx > 0) {
|
||||
scrape_engines_parallel(jobs, engine_idx);
|
||||
}
|
||||
|
||||
if (page == 1) {
|
||||
for (int i = 0; i < HANDLER_COUNT; i++) {
|
||||
@@ -301,10 +527,10 @@ int results_handler(UrlParams *params) {
|
||||
}
|
||||
|
||||
if (btnI) {
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
for (int i = 0; i < engine_idx; i++) {
|
||||
if (jobs[i].results_count > 0 && all_results[i][0].url) {
|
||||
char *redirect_url = strdup(all_results[i][0].url);
|
||||
for (int j = 0; j < ENGINE_COUNT; j++) {
|
||||
for (int j = 0; j < enabled_engine_count; j++) {
|
||||
for (int k = 0; k < jobs[j].results_count; k++) {
|
||||
free(all_results[j][k].url);
|
||||
free(all_results[j][k].title);
|
||||
@@ -327,7 +553,7 @@ int results_handler(UrlParams *params) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
for (int i = 0; i < enabled_engine_count; i++) {
|
||||
free(all_results[i]);
|
||||
}
|
||||
if (page == 1) {
|
||||
@@ -369,7 +595,7 @@ int results_handler(UrlParams *params) {
|
||||
}
|
||||
|
||||
int warning_count = 0;
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
for (int i = 0; i < enabled_engine_count; i++) {
|
||||
if (warning_message_for_job(&jobs[i]))
|
||||
warning_count++;
|
||||
}
|
||||
@@ -379,7 +605,7 @@ int results_handler(UrlParams *params) {
|
||||
int *warning_inner_counts = NULL;
|
||||
int warning_index = 0;
|
||||
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
for (int i = 0; i < enabled_engine_count; i++) {
|
||||
const char *warning_message = warning_message_for_job(&jobs[i]);
|
||||
if (!warning_message)
|
||||
continue;
|
||||
@@ -407,7 +633,7 @@ int results_handler(UrlParams *params) {
|
||||
}
|
||||
|
||||
int total_results = 0;
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
for (int i = 0; i < enabled_engine_count; i++) {
|
||||
total_results += jobs[i].results_count;
|
||||
}
|
||||
|
||||
@@ -427,7 +653,7 @@ int results_handler(UrlParams *params) {
|
||||
send_response(html);
|
||||
free(html);
|
||||
}
|
||||
for (int i = 0; i < ENGINE_COUNT; i++)
|
||||
for (int i = 0; i < enabled_engine_count; i++)
|
||||
free(all_results[i]);
|
||||
if (page == 1) {
|
||||
for (int i = 0; i < HANDLER_COUNT; i++) {
|
||||
@@ -441,7 +667,7 @@ int results_handler(UrlParams *params) {
|
||||
}
|
||||
int unique_count = 0;
|
||||
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
for (int i = 0; i < enabled_engine_count; i++) {
|
||||
for (int j = 0; j < jobs[i].results_count; j++) {
|
||||
char *display_url = all_results[i][j].url;
|
||||
|
||||
@@ -468,7 +694,7 @@ int results_handler(UrlParams *params) {
|
||||
continue;
|
||||
}
|
||||
results_matrix[unique_count] =
|
||||
(char **)malloc(sizeof(char *) * INFOBOX_FIELD_COUNT);
|
||||
(char **)malloc(sizeof(char *) * RESULT_FIELD_COUNT);
|
||||
if (!results_matrix[unique_count]) {
|
||||
free(seen_urls[unique_count]);
|
||||
free(all_results[i][j].url);
|
||||
@@ -477,6 +703,7 @@ int results_handler(UrlParams *params) {
|
||||
continue;
|
||||
}
|
||||
char *pretty_url = pretty_display_url(display_url);
|
||||
char *base_url = get_base_url(display_url);
|
||||
|
||||
results_matrix[unique_count][0] = strdup(display_url);
|
||||
results_matrix[unique_count][1] = strdup(pretty_url);
|
||||
@@ -486,10 +713,13 @@ int results_handler(UrlParams *params) {
|
||||
results_matrix[unique_count][3] =
|
||||
all_results[i][j].snippet ? strdup(all_results[i][j].snippet)
|
||||
: strdup("");
|
||||
results_matrix[unique_count][4] = strdup(base_url ? base_url : "");
|
||||
results_matrix[unique_count][5] = strdup("");
|
||||
|
||||
results_inner_counts[unique_count] = INFOBOX_FIELD_COUNT;
|
||||
results_inner_counts[unique_count] = RESULT_FIELD_COUNT;
|
||||
|
||||
free(pretty_url);
|
||||
free(base_url);
|
||||
free(all_results[i][j].url);
|
||||
free(all_results[i][j].title);
|
||||
free(all_results[i][j].snippet);
|
||||
@@ -502,6 +732,58 @@ int results_handler(UrlParams *params) {
|
||||
context_set_array_of_arrays(&ctx, "results", results_matrix, unique_count,
|
||||
results_inner_counts);
|
||||
|
||||
char ***pager_matrix = NULL;
|
||||
int *pager_inner_counts = NULL;
|
||||
int pager_count = 0;
|
||||
int pager_start = page <= 3 ? 1 : page - 2;
|
||||
int pager_end = pager_start + PAGER_WINDOW_SIZE - 1;
|
||||
|
||||
if (page > 3) {
|
||||
char *first_href = build_search_href(raw_query, selected_engine_id, 1);
|
||||
pager_count = add_link_to_collection(first_href, "First", "pagination-btn",
|
||||
&pager_matrix, &pager_inner_counts,
|
||||
pager_count);
|
||||
free(first_href);
|
||||
}
|
||||
|
||||
if (page > 1) {
|
||||
char *prev_href =
|
||||
build_search_href(raw_query, selected_engine_id, page - 1);
|
||||
pager_count = add_link_to_collection(prev_href, "Prev", "pagination-btn",
|
||||
&pager_matrix, &pager_inner_counts,
|
||||
pager_count);
|
||||
free(prev_href);
|
||||
}
|
||||
|
||||
for (int i = pager_start; i <= pager_end; i++) {
|
||||
char label[16];
|
||||
snprintf(label, sizeof(label), "%d", i);
|
||||
char *page_href = build_search_href(raw_query, selected_engine_id, i);
|
||||
pager_count = add_link_to_collection(
|
||||
page_href, label,
|
||||
i == page ? "pagination-btn pagination-current" : "pagination-btn",
|
||||
&pager_matrix, &pager_inner_counts, pager_count);
|
||||
free(page_href);
|
||||
}
|
||||
|
||||
char *next_href = build_search_href(raw_query, selected_engine_id, page + 1);
|
||||
pager_count = add_link_to_collection(next_href, "Next", "pagination-btn",
|
||||
&pager_matrix, &pager_inner_counts,
|
||||
pager_count);
|
||||
free(next_href);
|
||||
|
||||
if (pager_count > 0) {
|
||||
context_set_array_of_arrays(&ctx, "pagination_links", pager_matrix,
|
||||
pager_count, pager_inner_counts);
|
||||
for (int i = 0; i < pager_count; i++) {
|
||||
for (int j = 0; j < LINK_FIELD_COUNT; j++)
|
||||
free(pager_matrix[i][j]);
|
||||
free(pager_matrix[i]);
|
||||
}
|
||||
free(pager_matrix);
|
||||
free(pager_inner_counts);
|
||||
}
|
||||
|
||||
char *html = render_template("results.html", &ctx);
|
||||
if (html) {
|
||||
send_response(html);
|
||||
@@ -509,7 +791,7 @@ int results_handler(UrlParams *params) {
|
||||
}
|
||||
|
||||
for (int i = 0; i < unique_count; i++) {
|
||||
for (int j = 0; j < INFOBOX_FIELD_COUNT; j++)
|
||||
for (int j = 0; j < RESULT_FIELD_COUNT; j++)
|
||||
free(results_matrix[i][j]);
|
||||
free(results_matrix[i]);
|
||||
free(seen_urls[i]);
|
||||
@@ -524,7 +806,7 @@ int results_handler(UrlParams *params) {
|
||||
free(html);
|
||||
}
|
||||
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
for (int i = 0; i < enabled_engine_count; i++) {
|
||||
free(all_results[i]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ typedef int (*ParserFunc)(const char *engine_name, xmlDocPtr doc,
|
||||
SearchResult **out_results, int max_results);
|
||||
|
||||
typedef struct {
|
||||
const char *id;
|
||||
const char *name;
|
||||
const char *base_url;
|
||||
const char *host_header;
|
||||
@@ -24,6 +25,7 @@ typedef struct {
|
||||
int page_multiplier;
|
||||
int page_base;
|
||||
ParserFunc parser;
|
||||
int enabled;
|
||||
} SearchEngine;
|
||||
|
||||
typedef struct {
|
||||
@@ -54,8 +56,9 @@ typedef struct {
|
||||
ScrapeStatus status;
|
||||
} ScrapeJob;
|
||||
|
||||
extern const SearchEngine ENGINE_REGISTRY[];
|
||||
extern SearchEngine ENGINE_REGISTRY[];
|
||||
extern const int ENGINE_COUNT;
|
||||
void apply_engines_config(const char *engines_str);
|
||||
|
||||
size_t write_memory_callback(void *contents, size_t size, size_t nmemb,
|
||||
void *userp);
|
||||
|
||||
@@ -310,38 +310,122 @@ static int parse_yahoo(const char *engine_name, xmlDocPtr doc,
|
||||
static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
|
||||
SearchResult **out_results, int max_results);
|
||||
|
||||
const SearchEngine ENGINE_REGISTRY[] = {
|
||||
{.name = "DuckDuckGo Lite",
|
||||
SearchEngine ENGINE_REGISTRY[] = {
|
||||
{.id = "ddg",
|
||||
.name = "DuckDuckGo Lite",
|
||||
.base_url = "https://lite.duckduckgo.com/lite/?q=",
|
||||
.host_header = "lite.duckduckgo.com",
|
||||
.referer = "https://lite.duckduckgo.com/",
|
||||
.page_param = "s",
|
||||
.page_multiplier = 30,
|
||||
.page_base = 0,
|
||||
.parser = parse_ddg_lite},
|
||||
{.name = "Startpage",
|
||||
.parser = parse_ddg_lite,
|
||||
.enabled = 1},
|
||||
{.id = "startpage",
|
||||
.name = "Startpage",
|
||||
.base_url = "https://www.startpage.com/sp/search?q=",
|
||||
.host_header = "www.startpage.com",
|
||||
.referer = "https://www.startpage.com/",
|
||||
.page_param = "page",
|
||||
.page_multiplier = 1,
|
||||
.page_base = 1,
|
||||
.parser = parse_startpage},
|
||||
{.name = "Yahoo",
|
||||
.parser = parse_startpage,
|
||||
.enabled = 1},
|
||||
{.id = "yahoo",
|
||||
.name = "Yahoo",
|
||||
.base_url = "https://search.yahoo.com/search?p=",
|
||||
.host_header = "search.yahoo.com",
|
||||
.referer = "https://search.yahoo.com/",
|
||||
.page_param = "b",
|
||||
.page_multiplier = 10,
|
||||
.page_base = 1,
|
||||
.parser = parse_yahoo},
|
||||
{.name = "Mojeek",
|
||||
.parser = parse_yahoo,
|
||||
.enabled = 1},
|
||||
{.id = "mojeek",
|
||||
.name = "Mojeek",
|
||||
.base_url = "https://www.mojeek.com/search?q=",
|
||||
.host_header = "www.mojeek.com",
|
||||
.referer = "https://www.mojeek.com/",
|
||||
.page_param = "s",
|
||||
.page_multiplier = 10,
|
||||
.page_base = 1,
|
||||
.parser = parse_mojeek}};
|
||||
.parser = parse_mojeek,
|
||||
.enabled = 1}};
|
||||
|
||||
const int ENGINE_COUNT = sizeof(ENGINE_REGISTRY) / sizeof(SearchEngine);
|
||||
|
||||
static int engine_id_compare(const char *engine_id, const char *config_id) {
|
||||
while (*engine_id && *config_id) {
|
||||
char e = *engine_id;
|
||||
char c = *config_id;
|
||||
if (e >= 'A' && e <= 'Z')
|
||||
e = e - 'A' + 'a';
|
||||
if (c >= 'A' && c <= 'Z')
|
||||
c = c - 'A' + 'a';
|
||||
if (e != c)
|
||||
return 0;
|
||||
engine_id++;
|
||||
config_id++;
|
||||
}
|
||||
return *engine_id == *config_id;
|
||||
}
|
||||
|
||||
void apply_engines_config(const char *engines_str) {
|
||||
if (!engines_str || engines_str[0] == '\0') {
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
ENGINE_REGISTRY[i].enabled = 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
ENGINE_REGISTRY[i].enabled = 0;
|
||||
}
|
||||
|
||||
char *copy = strdup(engines_str);
|
||||
if (!copy)
|
||||
return;
|
||||
|
||||
char *saveptr;
|
||||
char *token = strtok_r(copy, ",", &saveptr);
|
||||
|
||||
while (token) {
|
||||
while (*token == ' ' || *token == '\t')
|
||||
token++;
|
||||
|
||||
if (strcmp(token, "*") == 0) {
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
ENGINE_REGISTRY[i].enabled = 1;
|
||||
}
|
||||
} else if (token[0] == '-' && token[1] != '\0') {
|
||||
char *engine_id = token + 1;
|
||||
int found = 0;
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
if (engine_id_compare(ENGINE_REGISTRY[i].id, engine_id)) {
|
||||
ENGINE_REGISTRY[i].enabled = 0;
|
||||
found = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
fprintf(stderr, "[WARN] Unknown engine: %s\n", engine_id);
|
||||
}
|
||||
} else {
|
||||
int found = 0;
|
||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||
if (engine_id_compare(ENGINE_REGISTRY[i].id, token)) {
|
||||
ENGINE_REGISTRY[i].enabled = 1;
|
||||
found = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
fprintf(stderr, "[WARN] Unknown engine: %s\n", token);
|
||||
}
|
||||
}
|
||||
|
||||
token = strtok_r(NULL, ",", &saveptr);
|
||||
}
|
||||
|
||||
free(copy);
|
||||
}
|
||||
|
||||
@@ -25,6 +25,12 @@ char *pretty_display_url(const char *input) {
|
||||
strncpy(temp, start, sizeof(temp) - 1);
|
||||
temp[sizeof(temp) - 1] = '\0';
|
||||
|
||||
char *query = strchr(temp, '?');
|
||||
if (query) {
|
||||
*query = '\0';
|
||||
input_len = strlen(temp);
|
||||
}
|
||||
|
||||
if (input_len > 0 && temp[input_len - 1] == '/') {
|
||||
temp[input_len - 1] = '\0';
|
||||
}
|
||||
|
||||
@@ -272,9 +272,69 @@ h1 span {
|
||||
gap:60px;
|
||||
padding:30px 60px;
|
||||
}
|
||||
.result-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-bottom: 2px;
|
||||
position: relative;
|
||||
}
|
||||
.result-favicon {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
flex-shrink: 0;
|
||||
background-size: cover;
|
||||
background-position: center;
|
||||
position: absolute;
|
||||
left: -24px;
|
||||
}
|
||||
.url {
|
||||
color: var(--text-secondary);
|
||||
font-size: 0.85rem;
|
||||
display: block;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.result-favicon {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
left: -20px;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 480px) {
|
||||
.result-favicon {
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
left: -16px;
|
||||
}
|
||||
}
|
||||
.results-container {
|
||||
grid-column:2;
|
||||
}
|
||||
.engine-filter-list {
|
||||
display:flex;
|
||||
flex-wrap:wrap;
|
||||
gap:10px;
|
||||
margin-bottom:24px;
|
||||
}
|
||||
.engine-filter {
|
||||
background:var(--bg-card);
|
||||
color:var(--text-secondary);
|
||||
border:1px solid var(--border);
|
||||
border-radius:999px;
|
||||
padding:6px 12px;
|
||||
text-decoration:none;
|
||||
font-size:0.85rem;
|
||||
font-weight:600;
|
||||
}
|
||||
.engine-filter.active {
|
||||
background:var(--accent);
|
||||
border-color:var(--accent);
|
||||
color:var(--bg-main);
|
||||
}
|
||||
.engine-warning-list {
|
||||
display:flex;
|
||||
flex-direction:column;
|
||||
@@ -308,12 +368,6 @@ h1 span {
|
||||
display:inline-block;
|
||||
margin-bottom:4px;
|
||||
}
|
||||
.url {
|
||||
color:var(--text-secondary);
|
||||
font-size:0.85rem;
|
||||
display:block;
|
||||
margin-bottom:4px;
|
||||
}
|
||||
.desc {
|
||||
color:var(--text-muted);
|
||||
line-height:1.6;
|
||||
@@ -407,25 +461,22 @@ h1 span {
|
||||
|
||||
|
||||
.pagination-current {
|
||||
background: var(--bg-card);
|
||||
color: var(--text-primary);
|
||||
border: 1px solid var(--border);
|
||||
padding: 4px 12px;
|
||||
border-radius: 8px;
|
||||
text-decoration: none;
|
||||
font-size: 1.2rem;
|
||||
font-weight: 600;
|
||||
transition: all 0.2s;
|
||||
touch-action: manipulation;
|
||||
background: var(--accent);
|
||||
border-color: var(--accent);
|
||||
color: var(--bg-main);
|
||||
}
|
||||
|
||||
.pagination-current:hover {
|
||||
background: var(--border);
|
||||
border-color: var(--text-secondary);
|
||||
background: var(--accent);
|
||||
border-color: var(--accent);
|
||||
}
|
||||
|
||||
|
||||
@media (max-width:1200px) {
|
||||
body {
|
||||
padding-left: 16px;
|
||||
padding-right: 16px;
|
||||
}
|
||||
.content-layout {
|
||||
grid-template-columns:1fr;
|
||||
padding:20px 30px;
|
||||
@@ -447,6 +498,10 @@ h1 span {
|
||||
}
|
||||
|
||||
@media (max-width:768px) {
|
||||
body {
|
||||
padding-left: 16px;
|
||||
padding-right: 16px;
|
||||
}
|
||||
header {
|
||||
flex-direction:column;
|
||||
gap:12px;
|
||||
|
||||
@@ -20,13 +20,14 @@
|
||||
Omni<span>Search</span>
|
||||
</h1>
|
||||
<form action="/search" method="GET" class="search-form">
|
||||
<input name="engine" type="hidden" value="{{selected_engine}}">
|
||||
<input name="q" type="text" class="search-box" autocomplete="off" placeholder="Search the web..."
|
||||
value="{{query}}">
|
||||
</form>
|
||||
</header>
|
||||
<nav class="nav-tabs">
|
||||
<div class="nav-container">
|
||||
<a href="/search?q={{query}}" class="active">
|
||||
<a href="{{search_href}}" class="active">
|
||||
All
|
||||
</a>
|
||||
<a href="/images?q={{query}}">
|
||||
@@ -38,6 +39,16 @@
|
||||
<aside class="sidebar-spacer">
|
||||
</aside>
|
||||
<main class="results-container">
|
||||
{{if exists engine_filters}}
|
||||
<nav class="engine-filter-list">
|
||||
{{for filter in engine_filters}}
|
||||
<a href="{{filter[0]}}" class="{{filter[2]}}">
|
||||
{{filter[1]}}
|
||||
</a>
|
||||
{{endfor}}
|
||||
</nav>
|
||||
{{endif}}
|
||||
|
||||
{{if exists engine_warnings}}
|
||||
<section class="engine-warning-list">
|
||||
{{for warning in engine_warnings}}
|
||||
@@ -55,9 +66,15 @@
|
||||
|
||||
{{for result in results}}
|
||||
<div class="result">
|
||||
<span class="url">
|
||||
{{result[1]}}
|
||||
<div class="result-header">
|
||||
<div class="result-favicon"
|
||||
style="background-image: url('https://{{result[4]}}/favicon.ico'), url('https://{{result[4]}}/favicon.png');">
|
||||
</div>
|
||||
<span class="url">
|
||||
{{result[1]}}
|
||||
</span>
|
||||
</span>
|
||||
</div>
|
||||
<a href="{{result[0]}}">
|
||||
{{result[2]}}
|
||||
</a>
|
||||
@@ -70,48 +87,15 @@
|
||||
</div>
|
||||
{{endfor}}
|
||||
|
||||
{{if exists pagination_links}}
|
||||
<nav class="pagination">
|
||||
<a class="pagination-btn prev" href="/search?q={{query}}&p={{prev_page}}">
|
||||
←
|
||||
</a>
|
||||
|
||||
{{if two_prev_page != 0}}
|
||||
<a class="pagination-btn prev" href="/search?q={{query}}&p={{two_prev_page}}">
|
||||
{{two_prev_page}}
|
||||
</a>
|
||||
{{endif}}
|
||||
|
||||
{{if prev_page != 0}}
|
||||
<a class="pagination-btn prev" href="/search?q={{query}}&p={{prev_page}}">
|
||||
{{prev_page}}
|
||||
</a>
|
||||
{{endif}}
|
||||
|
||||
<a class="pagination-current" href="/search?q={{query}}&p={{page}}">
|
||||
{{page}}
|
||||
</a>
|
||||
<a class="pagination-btn next" href="/search?q={{query}}&p={{next_page}}">
|
||||
{{next_page}}
|
||||
</a>
|
||||
<a class="pagination-btn next" href="/search?q={{query}}&p={{two_next_page}}">
|
||||
{{two_next_page}}
|
||||
</a>
|
||||
|
||||
{{if prev_page == 0}}
|
||||
<a class="pagination-btn prev" href="/search?q={{query}}&p=4">
|
||||
4
|
||||
</a>
|
||||
{{endif}}
|
||||
|
||||
{{if two_prev_page == 0}}
|
||||
<a class="pagination-btn prev" href="/search?q={{query}}&p=5">
|
||||
5
|
||||
</a>
|
||||
{{endif}}
|
||||
<a class="pagination-btn next" href="/search?q={{query}}&p={{next_page}}">
|
||||
→
|
||||
{{for link in pagination_links}}
|
||||
<a class="{{link[2]}}" href="{{link[0]}}">
|
||||
{{link[1]}}
|
||||
</a>
|
||||
{{endfor}}
|
||||
</nav>
|
||||
{{endif}}
|
||||
</main>
|
||||
<aside class="infobox-sidebar">
|
||||
{{if exists infoboxes}}
|
||||
|
||||
Reference in New Issue
Block a user