OSDN -- 開發和下載開源軟體
繁體中文翻譯狀態 translation status for zh-tw
  • R/O
  • SSH
  • HTTPS

ogup: 提交


Commit MetaInfo

修訂56 (tree)
時間2022-10-27 04:36:40
作者mateuszviste

Log Message

frozen ver 20221026 to tags

Change Summary

差異

--- tags/ogup-20221026/frontend/dbdump/gophermap.php (nonexistent)
+++ tags/ogup-20221026/frontend/dbdump/gophermap.php (revision 56)
@@ -0,0 +1,53 @@
1+<?php
2+
3+// db dump screen
4+
5+include '../db.php';
6+
7+$db = db_load('../ogupdb.dat', true, true, true);
8+$cnt = db_counters('../ogupdb.cnt');
9+
10+$dbtimestamp = gmdate("Y-m-d H:i", $cnt['timestamp']);
11+
12+echo "i\n";
13+echo "i >>> OGUP live database dump (as of {$dbtimestamp} UTC) <<<\n";
14+echo "i\n";
15+echo "7Apply a filter on hostname\t$\n";
16+echo "i\n";
17+if (!empty($_SERVER['QUERY_STRING_SEARCH'])) {
18+ $filter = trim($_SERVER['QUERY_STRING_SEARCH']);
19+ echo "iResults are filtered to hostnames matching '{$filter}'\n";
20+ echo "i\n";
21+}
22+echo "i ### | HOSTNAME | STATUS | NEXT CHECK (UTC) | LAST WORKING IP \n";
23+echo "i-----+---------------------------------+-----------------------+------------------+-----------------\n";
24+$i = 1;
25+foreach ($db as $item) {
26+ $host = $item['host'];
27+ if ($item['port'] != 70) $host .= ':' . $item['port'];
28+ if ((!empty($filter)) && (stripos($host, $filter) === FALSE)) continue;
29+
30+ $host = str_pad($host . ' ', 32, '.', STR_PAD_RIGHT);
31+
32+ if ($item['failedsince'] == 0) {
33+ $status = 'OK';
34+ } else if ($item['failedsince'] == 1) {
35+ $status = 'pending verification';
36+ } else {
37+ $status = 'down ' . gmdate("Y-m-d H:i", $item['failedsince']);
38+ }
39+ if ($item['nextcheck'] == 0) {
40+ $nextcheck = "ASAP";
41+ } else {
42+ $nextcheck = gmdate("Y-m-d H:i", $item['nextcheck']);
43+ }
44+ $status = str_pad($status, 22);
45+ $nextcheck = str_pad($nextcheck, 16);
46+ $id = str_pad($i, 4, ' ', STR_PAD_LEFT);
47+ echo "i{$id} | {$host}| {$status}| {$nextcheck} | {$item['ipaddr']}\n";
48+ $i++;
49+}
50+
51+echo "i-----+---------------------------------+-----------------------+------------------+-----------------\n";
52+
53+?>
--- tags/ogup-20221026/frontend/down/gophermap.php (nonexistent)
+++ tags/ogup-20221026/frontend/down/gophermap.php (revision 56)
@@ -0,0 +1,83 @@
1+<?php
2+
3+//
4+// down servers menu
5+//
6+// Copyright (C) 2019-2021 Mateusz Viste
7+//
8+
9+include '../db.php';
10+
11+function gen_unit($count, $name) {
12+ $res = '';
13+ if ($count > 0) {
14+ $res .= "{$count} {$name}";
15+ if ($count > 1) $res .= 's';
16+ $res .= ' ';
17+ }
18+ return($res);
19+}
20+
21+function time_diff_human($sec) {
22+ $MINUTE = 60;
23+ $HOUR = $MINUTE * 60;
24+ $DAY = $HOUR * 24;
25+ $WEEK = $DAY * 7;
26+ for ($w = 0; $sec >= $WEEK; $sec -= $WEEK) $w++;
27+ for ($d = 0; $sec >= $DAY; $sec -= $DAY) $d++;
28+ for ($h = 0; $sec >= $HOUR; $sec -= $HOUR) $h++;
29+ for ($m = 0; $sec >= $MINUTE; $sec -= $MINUTE) $m++;
30+ /* */
31+ if ($w > 0) {
32+ $res = gen_unit($w, 'week');
33+ if ($w < 10) $res .= gen_unit($d, 'day');
34+ } else if ($d > 0) {
35+ $res = gen_unit($d, 'day');
36+ $res .= gen_unit($h, 'hour');
37+ } else if ($h > 0) {
38+ $res = gen_unit($h, 'hour');
39+ $res .= gen_unit($m, 'min');
40+ } else if ($m > 0) {
41+ $res = gen_unit($m, 'minute');
42+ } else {
43+ $res = gen_unit($sec, 'second');
44+ }
45+ return(trim($res));
46+}
47+
48+
49+// custom sort of db items: failedsince-freshest first
50+function mysortfunc($a, $b) {
51+ if ($a['failedsince'] < $b['failedsince']) return(1);
52+ if ($a['failedsince'] > $b['failedsince']) return(-1);
53+ return(0);
54+}
55+
56+
57+echo "i\n";
58+echo "iThis page lists gopherspace nodes that have been observed by the OGUP\n";
59+echo "ias down recently. All timestamps are given in terran UTC time.\n";
60+echo "i\n";
61+
62+$db = db_load('../ogupdb.dat', false, false, true);
63+usort($db, mysortfunc);
64+
65+if (count($db) > 0) {
66+ echo "iGOPHER NODE DOWN SINCE\n";
67+} else {
68+ echo "iNothing to display. All known gopher servers appear to be healthy.\n";
69+}
70+
71+foreach ($db as $item) {
72+ $host = str_pad($item['host'] . ' ', 32, '.', STR_PAD_RIGHT);
73+ $port = $item['port'];
74+ $since = time_diff_human(time() - $item['failedsince']);
75+ $sincedate = gmdate("Y-m-d H:i", $item['failedsince']);
76+ $hostdownstr = str_pad($since, 16);
77+ echo "i{$host} {$hostdownstr} ({$sincedate})\n";
78+}
79+
80+echo "i\n";
81+echo "i---------------------------------------------------------- [EOF] ---\n";
82+
83+?>
--- tags/ogup-20221026/frontend/hist/gophermap.php (nonexistent)
+++ tags/ogup-20221026/frontend/hist/gophermap.php (revision 56)
@@ -0,0 +1,128 @@
1+<?php
2+
3+//
4+// OGUP servers history over time gophermap
5+//
6+// Copyright (C) 2019-2022 Mateusz Viste
7+//
8+
9+define (GRAPH_H, 21);
10+define (GRAPH_COL_MONTHS, 1); // how many months are contained in a single column
11+
12+
13+// counts all "up" servers in an xz file (servers with "failedsince = 0")
14+function count_up_servers_in_file($f) {
15+ $count = 0;
16+ $list = array();
17+
18+ $cmd = 'xzcat';
19+ if (strstr($f, ".gz") !== false) $cmd = 'zcat';
20+
21+ exec($cmd . ' ' . $f, $list);
22+ foreach ($list as $l) {
23+ $csv = str_getcsv($l);
24+ if (intval($csv[2]) != 0) continue; // skip failed servers
25+ $count++;
26+ }
27+ return($count);
28+}
29+
30+// get count of active gopher servers since Jan 2019
31+function get_history() {
32+ $curtime = date('Y') * 12 + date('m');
33+ $res = array();
34+
35+ for ($i = 2019 * 12; $i <= $curtime; $i += GRAPH_COL_MONTHS) {
36+ $yr = intval($i / 12);
37+ $mo = intval($i % 12) + 1;
38+ $dy = 1;
39+
40+ // dirty hack to fill some missing data...
41+ if (($yr == 2019) && ($mo == 1)) $mo = 2;
42+ if (($yr == 2020) && ($mo == 12)) $mo = 11;
43+ if (($yr == 2021) && ($mo < 11)) $mo = 11;
44+
45+ if (($yr == 2021) && ($mo == 11)) $dy = 27;
46+ if (($yr == 2019) && ($mo == 2)) $dy = 19;
47+
48+ $fname = sprintf('../db/ogupdb-%04d%02d%02d.dat.', $yr, $mo, $dy);
49+ if ($yr > 2020) {
50+ $fname .= 'xz';
51+ } else {
52+ $fname .= 'gz';
53+ }
54+ $res[$i] = count_up_servers_in_file($fname);
55+ }
56+ return $res;
57+}
58+
59+
60+$stats = get_history();
61+
62+echo "i\n";
63+echo "iThis histogram presents the amount of active gopher nodes over time, as\n";
64+echo "imeasured by the Observable Gopherspace Universe Project.\n";
65+echo "i\n";
66+
67+// find highest point for scaling graphs
68+$highest = 0;
69+foreach ($stats as $l) {
70+ if ($l > $highest) $highest = $l;
71+}
72+
73+/* build a 2D graph in an array */
74+$graph = array();
75+$column = 0;
76+//echo "istats = " . count($stats) . " ; highest = " . $highest . "\n";
77+foreach ($stats as $dat => $count) {
78+ //echo "i{$dat} => {$count}\n";
79+ // normalize count to a number of "pixels"
80+ $count = $count * GRAPH_H / $highest;
81+
82+ // mark vertical pixels
83+ for ($y = 0; $y < $count; $y++) {
84+ $graph[$y][$column] = 1;
85+ }
86+
87+ $column++;
88+}
89+
90+/* print out the graph */
91+$endgraphlimit = $column;
92+while ($endgraphlimit % (12 / GRAPH_COL_MONTHS)) $endgraphlimit++;
93+
94+for ($y = 0; $y < GRAPH_H; $y++) {
95+ echo 'i';
96+ for ($x = 0; $x <= $endgraphlimit; $x++) {
97+ /* add a space on year boundaries */
98+ if (($x % (12 / GRAPH_COL_MONTHS)) == 0) {
99+ if ($y & 1) {
100+ echo ' ';
101+ } else {
102+ echo '_';
103+ }
104+ }
105+ if ($graph[GRAPH_H - 1 - $y][$x] == 1) {
106+ echo '#';
107+ } else {
108+ if ($y & 1) {
109+ echo ' ';
110+ } else {
111+ echo '_';
112+ }
113+ }
114+ }
115+ if (($y & 1) == 0) echo intval($highest * (GRAPH_H - 1 - $y) / (GRAPH_H - 1));
116+ echo "\n";
117+}
118+
119+/* print out years */
120+$prevyr = 0;
121+echo "i";
122+foreach ($stats as $dat => $val) {
123+ $yr = intval($dat / 12);
124+ if ($yr != $prevyr) echo ' ' . str_pad($yr, (12 / GRAPH_COL_MONTHS), ' ', STR_PAD_BOTH);
125+ $prevyr = $yr;
126+}
127+
128+?>
--- tags/ogup-20221026/frontend/list/gophermap.php (nonexistent)
+++ tags/ogup-20221026/frontend/list/gophermap.php (revision 56)
@@ -0,0 +1,54 @@
1+<?php
2+
3+//
4+// OGUP servers list gophermap
5+//
6+// Copyright (C) 2019-2022 Mateusz Viste
7+//
8+
9+
10+include '../db.php';
11+
12+$cnt = db_counters('../ogupdb.cnt');
13+
14+$cnt_tot = str_pad($cnt['total'], 6, ' ', STR_PAD_LEFT);
15+$cnt_val = str_pad($cnt['up'], 6, ' ', STR_PAD_LEFT);
16+$cnt_unk = str_pad($cnt['pending'], 6, ' ', STR_PAD_LEFT);
17+$cnt_dwn = str_pad($cnt['down'], 6, ' ', STR_PAD_LEFT);
18+
19+echo "i\n";
20+echo "iThe Observable Gopherspace Universe Project knows about:\n";
21+echo "i{$cnt_tot} servers total, of which:\n";
22+echo "i{$cnt_val} have been validated as operational\n";
23+echo "i{$cnt_dwn} are down\n";
24+echo "i{$cnt_unk} are pending validation\n";
25+echo "i\n";
26+echo "iLast update: " . gmdate("Y-m-d H:i", $cnt['timestamp']) . " UTC\n";
27+echo "i\n";
28+
29+// list of servers
30+$db = db_load('../ogupdb.dat', true, false, false);
31+
32+if (! $db) {
33+ echo "3FAILED TO OPEN DB FILE\n";
34+} else {
35+ $buff = array();
36+ foreach ($db as $item) {
37+ if ($item['port'] != 70) {
38+ $nicename = $item['host'] . ':' . $item['port'];
39+ } else {
40+ $nicename = $item['host'];
41+ }
42+ //echo "1{$nicename}\t\t{$item['host']}\t{$item['port']}\n";
43+ $buff[] = "1{$nicename}\t\t{$item['host']}\t{$item['port']}";
44+ }
45+ // sort the output buffer and... output it
46+ sort($buff);
47+ foreach ($buff as $line) echo $line . "\n";
48+ fclose($fh);
49+}
50+
51+echo "i\n";
52+echo "i----------------------------- [EOF] ---\n";
53+
54+?>
--- tags/ogup-20221026/frontend/rnd/randlist.php (nonexistent)
+++ tags/ogup-20221026/frontend/rnd/randlist.php (revision 56)
@@ -0,0 +1,22 @@
1+<?php
2+
3+//
4+// a few random servers
5+//
6+// Copyright (C) 2019-2021 Mateusz Viste
7+//
8+
9+include '../db.php';
10+
11+$db = db_load('../ogupdb.dat', true, false, false);
12+shuffle($db);
13+
14+for ($i = 0; $i < 5; $i++) {
15+ $host = $db[$i]['host'];
16+ $port = $db[$i]['port'];
17+ $nice = $host;
18+ if ($port != 70) $nice .= ':' . $port;
19+ echo "1{$nice}\t\t{$host}\t{$port}\n";
20+}
21+
22+?>
--- tags/ogup-20221026/frontend/submit/gophermap.php (nonexistent)
+++ tags/ogup-20221026/frontend/submit/gophermap.php (revision 56)
@@ -0,0 +1,59 @@
1+<?php
2+
3+$serv = '';
4+if (!empty($_SERVER['QUERY_STRING_SEARCH'])) {
5+ $serv = strtolower(trim($_SERVER['QUERY_STRING_SEARCH']));
6+}
7+
8+// no query? output default screen
9+
10+if (empty($serv)) {
11+ echo "i\r\n";
12+ echo "iThe Observable Gopherspace Universe Project actively scans the universe\r\n";
13+ echo "ilooking for new gopher nodes, but it still may miss some. Help us by\r\n";
14+ echo "iproviding suggestions!\r\n";
15+ echo "i\r\n";
16+ echo "iSubmitting a new gopher server suggestion is easy: use the form below to\r\n";
17+ echo "ienter the address of the server you'd like to add to the OGUP database.\r\n";
18+ echo "iThat's it! Within a couple of hours OGUP will verify and it add to the\r\n";
19+ echo "ilist of known servers.\r\n";
20+ echo "i\r\n";
21+ echo "7Submit a gopher server to the OGUP (format: \"gopher.example.net\")\t?\r\n";
22+ echo "i\r\n";
23+ echo "iNotes:\r\n";
24+ echo "i * Only suggestions for server names or IP addresses are accepted (no paths).\r\n";
25+ echo "i * For a gopher server to be accepted, it must present at least one link in\r\n";
26+ echo "i its main menu that points to a self-hosted page.\r\n";
27+} else {
28+
29+ // process submission
30+
31+ sleep(2); // just to avoid someone hammers the server with some stupid script
32+
33+ echo "iYou have submitted the following input: \"{$serv}\"\r\n";
34+ echo "i\r\n";
35+
36+ if ((preg_match('/[a-z0-9][a-z0-9.:]*/', $serv) != 1) || (preg_match('/.*\..*/', $serv) != 1)) {
37+ echo "3ERROR: INVALID INPUT.\r\n";
38+ echo "i\r\n";
39+ echo "iSubmissions:\r\n";
40+ echo "i - must not contain characters other than a-z, 0-9, '.' and ':'.\r\n";
41+ echo "i - must start with either an a-z or 0-9 character\r\n";
42+ echo "i - must contain at least one dot\r\n";
43+ } else {
44+ $fname = md5($serv);
45+ if (file_exists('/tmp/ogup/' . $fname)) {
46+ echo "3ERROR: This entry has been already submitted.\r\n";
47+ } else {
48+ mkdir('/tmp/ogup'); // create if does not exist yet
49+ file_put_contents('/tmp/ogup/' . $fname, $serv);
50+ echo "iYour submission has been accepted. Thank you!\r\n";
51+ }
52+ }
53+
54+}
55+
56+echo "i\r\n";
57+echo "1Go back to OGUP main screen\t../\r\n";
58+
59+?>
--- tags/ogup-20221026/frontend/db.php (nonexistent)
+++ tags/ogup-20221026/frontend/db.php (revision 56)
@@ -0,0 +1,61 @@
1+<?php
2+
3+//
4+// db-related functions used by the PHP frontend
5+//
6+// Copyright (C) 2019-2021 Mateusz Viste
7+//
8+
9+// returns array with 5 counters: total, up, pending, down, timestamp
10+function db_counters($f) {
11+ $handle = fopen($f, "r");
12+ if ($handle === FALSE) return(FALSE);
13+ $csv = fgetcsv($handle, 64, ',');
14+ fclose($handle);
15+ // ogup.cnt is a CSV file with 4 values: count, countactive, countpending, countdown
16+ $res = array();
17+ $res['total'] = $csv[0];
18+ $res['up'] = $csv[1];
19+ $res['pending'] = $csv[2];
20+ $res['down'] = $csv[3];
21+ $res['timestamp'] = stat($f)['mtime'];
22+
23+ return($res);
24+}
25+
26+
27+// load the db into an array of items:
28+// ['host']
29+// ['port']
30+// ['failedsince'] => timestamp since failure (0=up, 1=pending)
31+// ['nextcheck'] => time when server should be checked again
32+// ['ipaddr'] => last known (working) IP
33+function db_load($f, $include_up, $include_pending, $include_down) {
34+ $fh = fopen('../ogupdb.dat', 'r');
35+ if (! $fh) return(FALSE);
36+
37+ // srv,port,downsince,nextcheck
38+ $res = array();
39+ while (($lin = fgetcsv($fh)) !== FALSE) {
40+ // include down?
41+ if ((intval($lin[2]) > 1) && (!$include_down)) continue;
42+ // include pending?
43+ if ((intval($lin[2]) == 1) && (!$include_pending)) continue;
44+ // include up?
45+ if ((intval($lin[2]) == 0) && (!$include_up)) continue;
46+ // add to array
47+ $item = array();
48+ $item['host'] = $lin[0];
49+ $item['port'] = intval($lin[1]);
50+ $item['failedsince'] = intval($lin[2]);
51+ $item['nextcheck'] = intval($lin[3]);
52+ $item['ipaddr'] = $lin[4];
53+ $res[] = $item;
54+ }
55+ fclose($fh);
56+
57+ return($res);
58+}
59+
60+
61+?>
--- tags/ogup-20221026/frontend/footer.php (nonexistent)
+++ tags/ogup-20221026/frontend/footer.php (revision 56)
@@ -0,0 +1,13 @@
1+<?php
2+
3+// OGUP main gophermap's stats footer
4+// Copyright (C) 2019-2021 Mateusz Viste
5+
6+include 'db.php';
7+
8+$cnt = db_counters('ogupdb.cnt');
9+$timestr = gmdate("Y-m-d", $cnt['timestamp']);
10+
11+echo "iAs of {$timestr}, the OGUP knows about {$cnt['up']} active servers.\n";
12+
13+?>
--- tags/ogup-20221026/gopherjoker/Makefile (nonexistent)
+++ tags/ogup-20221026/gopherjoker/Makefile (revision 56)
@@ -0,0 +1,17 @@
1+#
2+# build gopherjoker
3+#
4+# Observable Gopherspace Universe Project
5+# Copyright (C) 2019-2021 Mateusz Viste
6+#
7+
8+CC = clang
9+CFLAGS = -g -O2 -Wall -Wextra -pedantic -std=gnu89
10+CFLAGS += -Weverything -Wno-disabled-macro-expansion -Wno-padded
11+
12+all: gopherjoker
13+
14+gopherjoker: csv.o glist.o gopher.o gopherjoker.o
15+
16+clean:
17+ rm -f gopherjoker *.o
--- tags/ogup-20221026/gopherjoker/csv.c (nonexistent)
+++ tags/ogup-20221026/gopherjoker/csv.c (revision 56)
@@ -0,0 +1,45 @@
1+/*
2+ * CSV-reading C module
3+ *
4+ * Copyright (C) 2019 Mateusz Viste
5+ *
6+ */
7+
8+#include "csv.h"
9+
10+/* reads a csv line from file. returns 0 on success, -1 on EOF or error. */
11+int csv_readline(char *buf, unsigned short maxlinelen, char **ptrs, int maxptrs, FILE *fd) {
12+ int c;
13+ int i;
14+ int t;
15+ int blen = 0;
16+ /* read the line into buf */
17+ for (;;) {
18+ c = fgetc(fd);
19+ if ((c < 0) && (blen == 0)) return(-1);
20+ if ((c < 0) || (c == '\n')) break;
21+ if (c == '\r') continue;
22+ buf[blen++] = (char)c;
23+ if (blen == maxlinelen) return(-1);
24+ }
25+ buf[blen] = 0;
26+
27+ /* set up pointers and terminate fields */
28+ for (i = 0; i < maxptrs; i++) ptrs[i] = buf + blen; /* preinit to empty */
29+ ptrs[0] = buf;
30+ t = 1;
31+ for (i = 0;; i++) {
32+ switch (buf[i]) {
33+ case ',':
34+ buf[i] = 0;
35+ ptrs[t] = buf + i + 1;
36+ if ((t + 1) < maxptrs) t++;
37+ break;
38+ case '\r':
39+ case '\n':
40+ case 0:
41+ buf[i] = 0;
42+ return(0);
43+ }
44+ }
45+}
--- tags/ogup-20221026/gopherjoker/csv.h (nonexistent)
+++ tags/ogup-20221026/gopherjoker/csv.h (revision 56)
@@ -0,0 +1,16 @@
1+/*
2+ * CSV-reading C module
3+ *
4+ * Copyright (C) 2019 Mateusz Viste
5+ *
6+ */
7+
8+#ifndef CSV_H
9+#define CSV_H
10+
11+#include <stdio.h>
12+
13+/* reads a csv line from file. returns 0 on success, -1 on EOF or error. */
14+int csv_readline(char *buf, unsigned short maxlinelen, char **ptrs, int maxptrs, FILE *fd);
15+
16+#endif
--- tags/ogup-20221026/gopherjoker/glist.c (nonexistent)
+++ tags/ogup-20221026/gopherjoker/glist.c (revision 56)
@@ -0,0 +1,143 @@
1+/*
2+ * glist-handling routines
3+ * This file is part of the Obsevable Gopherspace Universe Project
4+ * Copyright (C) 2019-2021 Mateusz Viste
5+ */
6+
7+#include <ctype.h> /* tolower() */
8+#include <stdlib.h>
9+#include <string.h> /* strcpy() */
10+#include <time.h>
11+
12+#include "glist.h"
13+
14+
15+/* rotate an uint32_t value by 1 bit left */
16+static void rotl(uint32_t *u) {
17+ uint32_t bit;
18+ bit = *u & 1;
19+ *u <<= 1;
20+ *u |= bit;
21+}
22+
23+
24+/* compute hash of a glist node (ie. a host, port, selector tuple) - used to
25+ * determine the identity of a glist entry. this is a somewhat specialized
26+ * version of a BSD sum. */
27+uint32_t glist_node_hash(const struct gopherlist *node) {
28+ uint32_t res = 0;
29+ int i;
30+ /* start with port */
31+ res = node->port;
32+ /* add host (case-insensitive) */
33+ for (i = 0; node->fqdn[i] != 0; i++) {
34+ rotl(&res);
35+ res ^= (uint32_t)tolower(node->fqdn[i]);
36+ }
37+ /* add selector */
38+ for (i = 0; (node->selector != NULL) && (node->selector[i] != 0); i++) {
39+ rotl(&res);
40+ res ^= (uint32_t)node->selector[i];
41+ }
42+ return(res);
43+}
44+
45+
46+void glist_free(struct gopherlist *glist) {
47+ struct gopherlist *victim;
48+ while (glist) {
49+ free(glist->selector);
50+ victim = glist;
51+ glist = glist->next;
52+ free(victim);
53+ }
54+}
55+
56+
57+struct gopherlist *glist_findhostport(struct gopherlist *glist, const char *host, const unsigned short port) {
58+ struct gopherlist *node;
59+ for (node = glist; node != NULL; node = node->next) {
60+ if ((node->port == port) && (strcasecmp(node->fqdn, host) == 0)) return(node);
61+ }
62+ return(NULL);
63+}
64+
65+
66+/* allocate a gopherlist node with host:port filled in */
67+struct gopherlist *glist_node_alloc(const char *host, unsigned short port) {
68+ struct gopherlist *node;
69+
70+ /* alloc struct */
71+ node = calloc(1, sizeof(struct gopherlist) + strlen(host) + 1);
72+
73+ /* fill in fields */
74+ if (node != NULL) {
75+ strcpy(node->fqdn, host);
76+ node->port = port;
77+ }
78+
79+ return(node);
80+}
81+
82+
83+/* add new host to glist, unless said host:port pair already exists there.
84+ * returns pointer to the new (or already existing) struct. NULL on error. */
85+struct gopherlist *glist_addnewhostport(struct gopherlist **glist, const char *newhost, unsigned short newport, time_t failedsince) {
86+ struct gopherlist *node;
87+
88+ /* is entry in list already? */
89+ node = glist_findhostport(*glist, newhost, newport);
90+ if (node != NULL) return(node);
91+
92+ /* create new node */
93+ node = glist_node_alloc(newhost, newport);
94+ if (node == NULL) return(NULL);
95+
96+ /* fill failedsince */
97+ node->failedsince = failedsince;
98+
99+ /* attach the newly created node to glist */
100+ glist_node_chain(glist, node);
101+
102+ return(node);
103+}
104+
105+
106+void glist_node_free(struct gopherlist **node) {
107+ if (*node == NULL) return;
108+ free((*node)->selector);
109+ free(*node);
110+ *node = NULL;
111+}
112+
113+
114+/* insert node into a glist */
115+void glist_node_chain(struct gopherlist **glist, struct gopherlist *node) {
116+ node->prev = NULL;
117+ node->next = *glist;
118+ if (*glist != NULL) (*glist)->prev = node;
119+ *glist = node;
120+}
121+
122+
123+void glist_node_unchain(struct gopherlist **glist, struct gopherlist *node) {
124+ if (node->prev == NULL) {
125+ *glist = node->next;
126+ if (*glist != NULL) (*glist)->prev = NULL;
127+ } else {
128+ node->prev->next = node->next;
129+ if (node->next != NULL) node->next->prev = node->prev;
130+ }
131+}
132+
133+
134+struct gopherlist *glist_node_dup(struct gopherlist *node) {
135+ struct gopherlist *res;
136+ if (node == NULL) return(NULL);
137+ res = calloc(1, sizeof(struct gopherlist) + strlen(node->fqdn));
138+ if (res == NULL) return(NULL);
139+
140+ memcpy(res, node, sizeof(struct gopherlist) + strlen(node->fqdn));
141+ if (node->selector != NULL) res->selector = strdup(node->selector);
142+ return(res);
143+}
--- tags/ogup-20221026/gopherjoker/glist.h (nonexistent)
+++ tags/ogup-20221026/gopherjoker/glist.h (revision 56)
@@ -0,0 +1,49 @@
1+/*
2+ * glist-handling routines
3+ * This file is part of the Obsevable Gopherspace Universe Project
4+ * Copyright (C) 2019-2021 Mateusz Viste
5+ */
6+
7+#ifndef GLIST_H
8+#define GLIST_H
9+
10+#include <stdint.h>
11+
12+struct gopherlist {
13+ time_t failedsince;
14+ time_t nextcheck;
15+ struct gopherlist *next;
16+ struct gopherlist *prev;
17+ unsigned short port;
18+ char *selector;
19+ char ipaddr[64];
20+ char fqdn[1];
21+};
22+
23+
24+/* compute hash of a glist node (ie. a host, port, selector tuple) - used to
25+ * determine the identity of a glist entry. this is a somewhat specialized
26+ * version of a BSD sum. */
27+uint32_t glist_node_hash(const struct gopherlist *node);
28+
29+void glist_free(struct gopherlist *glist);
30+
31+struct gopherlist *glist_findhostport(struct gopherlist *glist, const char *host, const unsigned short port);
32+
33+/* allocate a gopherlist node with host:port filled in */
34+struct gopherlist *glist_node_alloc(const char *host, unsigned short port);
35+
36+/* add new host to glist, unless said host:port pair already exists there.
37+ * returns pointer to the new (or already existing) struct. NULL on error. */
38+struct gopherlist *glist_addnewhostport(struct gopherlist **glist, const char *newhost, unsigned short newport, time_t failedsince);
39+
40+void glist_node_free(struct gopherlist **node);
41+
42+/* insert node into a glist */
43+void glist_node_chain(struct gopherlist **glist, struct gopherlist *node);
44+
45+void glist_node_unchain(struct gopherlist **glist, struct gopherlist *node);
46+
47+struct gopherlist *glist_node_dup(struct gopherlist *node);
48+
49+#endif
--- tags/ogup-20221026/gopherjoker/gopher.c (nonexistent)
+++ tags/ogup-20221026/gopherjoker/gopher.c (revision 56)
@@ -0,0 +1,162 @@
1+/*
2+ * gopher-related routines
3+ * This file is part of the Obsevable Gopherspace Universe Project
4+ * Copyright (C) 2019-2021 Mateusz Viste
5+ */
6+
7+#include <arpa/inet.h> /* inet_pton() */
8+#include <errno.h>
9+#include <fcntl.h>
10+#include <netdb.h> /* gethostbyname() */
11+#include <stdlib.h> /* atoi() */
12+#include <string.h> /* strlen() */
13+#include <sys/types.h>
14+#include <sys/socket.h>
15+#include <time.h> /* ctime(), time_t */
16+#include <unistd.h> /* close() */
17+
18+
19+#include "gopher.h"
20+
21+
22+static int connect_nonblocking(int s, const struct sockaddr *addr, socklen_t addrlen, int timeout) {
23+ int r;
24+ struct timeval t;
25+ fd_set selset;
26+
27+ r = connect(s, addr, addrlen);
28+ if (r == 0) return(0);
29+ if (errno != EINPROGRESS) return(r);
30+ FD_ZERO(&selset);
31+ FD_SET(s, &selset);
32+ t.tv_sec = timeout;
33+ t.tv_usec = 0;
34+ r = select(s + 1, NULL, &selset, NULL, &t);
35+ if (r < 0) return(-3);
36+ r = connect(s, addr, addrlen);
37+ return(r);
38+}
39+
40+
41+long gopher_fetch(char *buff, size_t buffsz, const char *host, unsigned short port, const char *selector, char *ipstr, size_t ipstrsz) {
42+ int sock = -1;
43+ size_t len;
44+ int flags;
45+ time_t timeout;
46+ struct addrinfo *addr, *addrptr;
47+
48+ ipstr[0] = 0;
49+
50+ /* resolve host & connect */
51+ if (getaddrinfo(host, NULL, NULL, &addr) != 0) return(-1);
52+ for (addrptr = addr; addrptr != NULL; addrptr = addrptr->ai_next) {
53+ struct sockaddr_in *sin;
54+ sock = socket(addrptr->ai_family, addrptr->ai_socktype, addrptr->ai_protocol);
55+ if (sock < 0) continue;
56+ /* set port */
57+ sin = (void *)(addrptr->ai_addr);
58+ sin->sin_port = htons(port);
59+ /* set socket as non-blocking */
60+ flags = fcntl(sock, F_GETFL);
61+ fcntl(sock, F_SETFL, flags | O_NONBLOCK);
62+ /* try to connect */
63+ if (connect_nonblocking(sock, addrptr->ai_addr, addrptr->ai_addrlen, 10) == 0) {
64+ /* fill ipstr and continue */
65+ getnameinfo(addrptr->ai_addr, addrptr->ai_addrlen, ipstr, (socklen_t)ipstrsz, NULL, 0, NI_NUMERICHOST);
66+ break;
67+ } else { /* close sock and try next option (if any) */
68+ close(sock);
69+ sock = -1;
70+ }
71+ }
72+ freeaddrinfo(addr);
73+ if (sock < 0) return(-2);
74+
75+ /* send selector, terminated by a CR/LF pair */
76+ if (selector != NULL) send(sock, selector, strlen(selector), MSG_MORE);
77+ send(sock, "\r\n", 2, 0);
78+
79+ /* fetch answer until end of transmission or timeout */
80+ len = 0;
81+ timeout = time(NULL) + 10;
82+ for (;;) {
83+ size_t spaceleft = buffsz - len;
84+ ssize_t rlen;
85+ if (time(NULL) > timeout) {
86+ close(sock);
87+ return(-3);
88+ }
89+ if (spaceleft == 0) break; /* I'm stuffed thank you */
90+ rlen = recv(sock, buff + len, spaceleft, 0);
91+ if (rlen == 0) break; /* orderly shutdown */
92+ if (rlen < 0) { /* sock error */
93+ if ((errno == EWOULDBLOCK) || (errno == EAGAIN)) continue;
94+ close(sock);
95+ return(-4);
96+ }
97+ len += (size_t)rlen;
98+ }
99+
100+ /* close sock and quit */
101+ close(sock);
102+ return((long)len);
103+}
104+
105+
106+/* parse a gopher menu line entry and fill host, port and selector accordingly.
107+ * returns 0 on success, non-zero otherwise. */
108+int gopher_menu_parseline(char *host, unsigned short hostsz, unsigned short *port, char *selector, unsigned short selectorsz, const char *menu, long menulen) {
109+ char portstr[8];
110+ unsigned short i = 0, t;
111+ /* skip first tabs (description) */
112+ for (;;) {
113+ i++;
114+ /* printf("i=%d ['%c']\n", i, menu[i]); */
115+ if (menu[i] == '\t') break;
116+ if (menu[i] == '\r') return(-2);
117+ if (menu[i] == '\n') return(-3);
118+ if (i >= menulen) return(-4);
119+ }
120+ /* read selector */
121+ for (t = 0; ; t++) {
122+ if (t >= selectorsz) return(-5);
123+ i++;
124+ if (i >= menulen) return(-6);
125+ if (menu[i] == '\r') return(-7);
126+ if (menu[i] == '\n') return(-8);
127+ selector[t] = menu[i];
128+ if (selector[t] == '\t') {
129+ selector[t] = 0;
130+ break;
131+ }
132+ }
133+ /* read hostname */
134+ for (t = 0; ; t++) {
135+ if (t >= hostsz) return(-9);
136+ i++;
137+ if (i >= menulen) return(-10);
138+ if (menu[i] == '\r') return(-11);
139+ if (menu[i] == '\n') return(-12);
140+ host[t] = menu[i];
141+ if (host[t] == '\t') {
142+ host[t] = 0;
143+ break;
144+ }
145+ }
146+ /* read port */
147+ for (t = 0; ; t++) {
148+ if (t >= sizeof(portstr)) return(-13);
149+ i++;
150+ if (i >= menulen) return(-14);
151+ portstr[t] = menu[i];
152+ if ((portstr[t] == '\t') || (portstr[t] == '\n')) {
153+ int tport;
154+ portstr[t] = 0;
155+ tport = atoi(portstr);
156+ if ((tport < 1) || (tport > 0xffff)) return(-15);
157+ *port = (unsigned short)tport;
158+ break;
159+ }
160+ }
161+ return(0);
162+}
--- tags/ogup-20221026/gopherjoker/gopher.h (nonexistent)
+++ tags/ogup-20221026/gopherjoker/gopher.h (revision 56)
@@ -0,0 +1,16 @@
1+/*
2+ * gopher-related routines
3+ * This file is part of the Obsevable Gopherspace Universe Project
4+ * Copyright (C) 2019-2021 Mateusz Viste
5+ */
6+
7+#ifndef GOPHER_H
8+#define GOPHER_H
9+
10+long gopher_fetch(char *buff, size_t buffsz, const char *host, unsigned short port, const char *selector, char *ipstr, size_t ipstrsz);
11+
12+/* parse a gopher menu line entry and fill host, port and selector accordingly.
13+ * returns 0 on success, non-zero otherwise. */
14+int gopher_menu_parseline(char *host, unsigned short hostsz, unsigned short *port, char *selector, unsigned short selectorsz, const char *menu, long menulen);
15+
16+#endif
--- tags/ogup-20221026/gopherjoker/gopherjoker.c (nonexistent)
+++ tags/ogup-20221026/gopherjoker/gopherjoker.c (revision 56)
@@ -0,0 +1,626 @@
1+/*
2+ * gopherjoker, part of the Observable Gopherspace Universe Project
3+ * Copyright (C) 2019-2022 Mateusz Viste
4+ *
5+ * 2021-12-11: lazy scanning of the IPv4 space when time allows (--scanipv4)
6+ * 2021-12-09: a /tmp/ogup/... submission file may contain many entries
7+ * 2021-12-07: write last good IP address to database for every host
8+ * 2021-12-06: scheduler for probing servers + internal cache of recent checks
9+ * 2021-12-04: added --saveperiod and --waitperiod cmdline parameters
10+ * 2021-12-03: new hosts can be fed to gopherjoker through /tmp/ogup/
11+ * 2020-01-13: added recent history so joker won't revisit places too often
12+ * 2020-01-13: variety of minor fixes and casts to shut clang warnings
13+ * 2019-03-09: hosts with no menu entries are kept in db - but as IP only
14+ * 2019-02-27: host is removed if its main menu has no reference to itself
15+ * 2019-02-20: fixed null ptr dereference, added a 'down' count to countfile
16+ * 2019-02-19: countfile contains 3 values: total, active and pending servers
17+ * 2019-02-18: first public release
18+ *
19+ * This software is made available under the terms of the MIT License:
20+ *
21+ * Permission is hereby granted, free of charge, to any person obtaining a
22+ * copy of this software and associated documentation files (the "Software"),
23+ * to deal in the Software without restriction, including without limitation
24+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
25+ * and/or sell copies of the Software, and to permit persons to whom the
26+ * Software is furnished to do so, subject to the following conditions:
27+ *
28+ * The above copyright notice and this permission notice shall be included in
29+ * all copies or substantial portions of the Software.
30+ *
31+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
32+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
33+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
34+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
35+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
36+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
37+ * DEALINGS IN THE SOFTWARE.
38+ */
39+
40+#include <dirent.h> /* opendir() */
41+#include <errno.h>
42+#include <stdio.h> /* printf(), fopen(), ... */
43+#include <stdlib.h> /* calloc(), rand48() */
44+#include <string.h> /* strcpy() */
45+#include <time.h> /* ctime(), time_t */
46+#include <unistd.h> /* sleep() */
47+
48+#include "csv.h"
49+#include "glist.h"
50+#include "gopher.h"
51+
52+
53+#define WAITPERIOD_DEFAULT 20
54+#define SAVEPERIOD_DEFAULT 3600
55+#define NEWHOSTSDIR "/tmp/ogup/"
56+#define MAXFAILTIME (3600 * 24 * 60)
57+#define TTLINIT 64
58+#define CHECKPERIOD (3600 * 24)
59+#define CHECKPERIOD_FAST (3600 * 2)
60+#define NOJOB_SLEEP 60 /* how long to sleep if there is nothing to do */
61+
62+static int WAITPERIOD = WAITPERIOD_DEFAULT;
63+static int SAVEPERIOD = SAVEPERIOD_DEFAULT;
64+static int SCANIPV4SPACE = 0;
65+
66+
67+/**************** FUNCTIONS ****************/
68+
69+
70+/* return a human date/time string based on time_t timestamp */
71+static char *epoch2human(time_t timestamp) {
72+ static char buff[64];
73+ struct tm *t = gmtime(&timestamp);
74+ strftime(buff, sizeof(buff), "%F %R", t);
75+ return(buff);
76+}
77+
78+
79+/* returns pointer to dbfile name */
80+static int parseargs(int argc, char **argv, char **dbfile, char **dbcount) {
81+ int i;
82+ *dbfile = NULL;
83+ *dbcount = NULL;
84+ for (i = 1; i < argc; i++) {
85+ if (strcasecmp(argv[i], "--saveperiod") == 0) {
86+ i++;
87+ SAVEPERIOD = atoi(argv[i]);
88+ if (SAVEPERIOD < 1) {
89+ fprintf(stderr, "ERROR: --saveperiod must be a positive integer\n");
90+ return(-1);
91+ }
92+ } else if (strcasecmp(argv[i], "--waitperiod") == 0) {
93+ i++;
94+ WAITPERIOD = atoi(argv[i]);
95+ if (WAITPERIOD < 1) {
96+ fprintf(stderr, "ERROR: --waitperiod must be a positive integer\n");
97+ return(-1);
98+ }
99+ } else if (strcasecmp(argv[i], "--scanipv4") == 0) {
100+ SCANIPV4SPACE = 1;
101+ } else if ((argv[i][0] != '-') && ((*dbfile == NULL) || (*dbcount == NULL))) {
102+ if (*dbfile == NULL) {
103+ *dbfile = argv[i];
104+ } else {
105+ *dbcount = argv[i];
106+ }
107+ } else {
108+ /* unknown argument */
109+ return(-1);
110+ }
111+ }
112+ if ((*dbfile == NULL) || (*dbcount == NULL)) return(-2);
113+ return(0);
114+}
115+
116+
117+static struct gopherlist *loaddb(const char *fname) {
118+ FILE *fd;
119+ int i;
120+ char *ptrs[8];
121+ char lbuf[512];
122+ struct gopherlist *res = NULL, *newnode;
123+ fd = fopen(fname, "rb");
124+ if (fd == NULL) return(NULL);
125+ for (;;) {
126+ i = csv_readline(lbuf, sizeof(lbuf), ptrs, 6, fd);
127+ if (i != 0) break;
128+ /* TSV line structure
129+ * 0 hostname
130+ * 1 port
131+ * 2 failedsince (time_t)
132+ * 3 nextcheck (time_t)
133+ * 4 ipaddr (string) */
134+ newnode = glist_addnewhostport(&res, ptrs[0], (unsigned short)atoi(ptrs[1]), atol(ptrs[2]));
135+ if (newnode == NULL) {
136+ /* on error, free list and quit */
137+ glist_free(res);
138+ return(NULL);
139+ }
140+ newnode->nextcheck = atoi(ptrs[3]);
141+ strcpy(newnode->ipaddr, ptrs[4]);
142+ }
143+ fclose(fd);
144+ return(res);
145+}
146+
147+
148+static unsigned long savedb(const char *fname, const char *fcount, struct gopherlist *glist) {
149+ unsigned long count = 0, countactive = 0, countpending = 0, countdown = 0;
150+ FILE *f;
151+ struct gopherlist *node;
152+ f = fopen(fname, "wb");
153+ if (f == NULL) return(0);
154+ for (node = glist; node != NULL; node = node->next) {
155+ count++;
156+ if (node->failedsince == 0) countactive++;
157+ if (node->failedsince == 1) countpending++;
158+ if (node->failedsince > 1) countdown++;
159+ fprintf(f, "%s,%u,%ld,%ld,%s\n", node->fqdn, node->port, node->failedsince, node->nextcheck, node->ipaddr);
160+ }
161+ fclose(f);
162+ f = fopen(fcount, "wb");
163+ if (f == NULL) return(0);
164+ fprintf(f, "%lu,%lu,%lu,%lu\n", count, countactive, countpending, countdown);
165+ fclose(f);
166+ return(count);
167+}
168+
169+
170+/* open fname and read a server name from it, write it to *s. returns
171+ * port number on success, neg value on error
172+ * TODO support for IPv6 addresses like [2a60:1234::1]:71
173+ */
174+static void process_serverlist_file(struct gopherlist **glist, const char *fname) {
175+ FILE *f;
176+ int port;
177+ int i;
178+ char s[128];
179+
180+ /* open file */
181+ f = fopen(fname, "rb");
182+ if (f == NULL) {
183+ printf("ERROR: failed to open file %s (%s)\r\n", fname, strerror(errno));
184+ return;
185+ }
186+
187+ for (;;) {
188+ /* read server string */
189+ if (fgets(s, sizeof(s), f) == NULL) break;
190+
191+ /* parse (look for a port after ':' and trim \r and \n) */
192+ port = 70;
193+ for (i = 0; s[i] != 0; i++) {
194+ if ((s[i] == '\r') || (s[i] == '\n')) {
195+ s[i] = 0;
196+ break;
197+ }
198+ if (s[i] == ':') {
199+ s[i++] = 0;
200+ port = atoi(s + i);
201+ break;
202+ }
203+ }
204+
205+ if ((port < 1) || (port > 65535)) {
206+ printf("invalid port: host=%s port=%d\n", s, port);
207+ continue;
208+ }
209+
210+ /* add server to database */
211+ if (glist_addnewhostport(glist, s, (unsigned short)port, 1) != NULL) {
212+ printf("added host: %s:%d\r\n", s, port);
213+ } else {
214+ puts("glist_addnewhostport() call failed");
215+ }
216+
217+ }
218+
219+ fclose(f);
220+}
221+
222+
223+/* insert any newly-subnmitted servers to memory database (and remove the new host file) */
224+static void loadextrahosts(struct gopherlist **glist, const char *dir) {
225+ char fname[128];
226+ DIR *d;
227+ struct dirent *dptr;
228+
229+ d = opendir(dir);
230+ if (d == NULL) return;
231+
232+ while ((dptr = readdir(d)) != NULL) {
233+ if (dptr->d_type != DT_REG) continue; /* skip anything that's not a regular file (includes symlinks) */
234+ strcpy(fname, dir);
235+ strcat(fname, "/");
236+ strcat(fname, dptr->d_name);
237+ printf("loading new host file: %s\n", fname);
238+ process_serverlist_file(glist, fname);
239+ /* remove file and go to next one */
240+ unlink(fname);
241+ }
242+ closedir(d);
243+}
244+
245+
246+/* pick a glist node that needs to be probed */
247+static struct gopherlist *pickhostfromlist(struct gopherlist *glist) {
248+ struct gopherlist *gptr, *res;
249+ time_t now = time(NULL);
250+
251+ if (glist == NULL) return(NULL);
252+
253+ /* find the most late entry */
254+ res = glist;
255+ for (gptr = glist; gptr != NULL; gptr = gptr->next) {
256+ if (gptr->nextcheck < res->nextcheck) res = gptr;
257+ }
258+
259+ /* is it late at all? */
260+ if (res->nextcheck > now) {
261+ printf("pickhostfromlist(): found no candidates to be checked (next candidate is %s:%u to be checked at %s\n", res->fqdn, res->port, epoch2human(res->nextcheck));
262+ return(NULL);
263+ }
264+
265+ if (res->nextcheck == 0) {
266+ printf("pickhostfromlist(): %s:%u needs to be checked (never checked yet)\n", res->fqdn, res->port);
267+ } else {
268+ printf("pickhostfromlist(): %s:%u needs to be checked (late by %ds)\n", res->fqdn, res->port, (int)(now - res->nextcheck));
269+ }
270+ return(res);
271+}
272+
273+
274+static struct gopherlist *pickrandhostfromlist(struct gopherlist *glist) {
275+ struct gopherlist *gptr;
276+ unsigned long cnt;
277+ unsigned long i;
278+
279+ if (glist == NULL) return(NULL);
280+
281+ /* count how many entries I have */
282+ cnt = 0;
283+ for (gptr = glist; gptr != NULL; gptr = gptr->next) cnt++;
284+
285+ /* random entry id */
286+ i = (unsigned long)mrand48() % cnt;
287+
288+ /* fast-forward to the given entry and return it */
289+ while (i--) glist = glist->next;
290+ return(glist);
291+}
292+
293+
294+static int ishostvalid(const char *host) {
295+ unsigned short i;
296+ for (i = 0; host[i] != 0; i++) {
297+ if ((host[i] >= 'a') && (host[i] <= 'z')) continue;
298+ if ((host[i] >= 'A') && (host[i] <= 'Z')) continue;
299+ if ((host[i] >= '0') && (host[i] <= '9')) continue;
300+ if (host[i] == '-') continue;
301+ if (host[i] == '.') continue;
302+ return(-1);
303+ }
304+ if (i < 3) return(-10); /* host len should be at least 3 chars long */
305+ return(0);
306+}
307+
308+
309+/* parses a gopher menu and remembers all the hosts present in its links */
310+static struct gopherlist *menu2gopherlist(const char *menu, long menulen) {
311+ long i;
312+ struct gopherlist *res = NULL;
313+ struct gopherlist *node;
314+ int n;
315+
316+ /* DEBUG */
317+ printf("*\t*\t*\t*\t*\t*\t*\n");
318+ n = 0;
319+ for (i = 0; i < menulen; i++) {
320+ printf("%c", menu[i]);
321+ if ((menu[i] == '\n') && (++n >= 8)) break; /* limit debug output to 8 lines */
322+ }
323+ printf("*\t*\t*\t*\t*\t*\t*\n");
324+
325+ /* iterate line by line */
326+ for (i = 0; i < menulen; i++) {
327+ char host[64];
328+ char selector[128];
329+ unsigned short port;
330+ char type;
331+
332+ /* skip to next line (unless I only started) */
333+ if (i > 0) {
334+ while ((i < menulen) && (menu[i] != '\n')) i++;
335+ while ((i < menulen) && (menu[i] == '\n')) i++;
336+ }
337+ if (i >= menulen) break;
338+
339+ type = menu[i];
340+
341+ if ((type != '0') && (type != '1')) continue;
342+
343+ if (gopher_menu_parseline(host, sizeof(host), &port, selector, sizeof(selector), menu + i, menulen - i) != 0) continue;
344+
345+ /* validate host name */
346+ if (ishostvalid(host) != 0) continue;
347+
348+ /* alloc a new node */
349+ node = glist_node_alloc(host, port);
350+ if (node == NULL) {
351+ glist_free(res);
352+ printf("ERR: OUT OF MEMORY\n");
353+ return(NULL);
354+ }
355+
356+ /* attach the newly created node to glist */
357+ glist_node_chain(&res, node);
358+
359+ if (menu[i] == '1') node->selector = strdup(selector);
360+ }
361+
362+ return(res);
363+}
364+
365+
366+/* a cached wrapper around gopher_fetch() */
367+static long cached_gopher_fetch(char *buff, size_t buffsz, const char *host, unsigned short port, const char *selector, char *ipstr, size_t ipstrsz) {
368+ long res;
369+ int i;
370+ char id[512];
371+ static unsigned short nextentry;
372+
373+ static struct goph_cache {
374+ char *id;
375+ char *ipstr;
376+ char *data;
377+ long bytes;
378+ } CACHE[TTLINIT];
379+
380+ /* compute a string that contains the host/port/selector tuple */
381+ snprintf(id, sizeof(id), "%s|%u|%s", host, port, (selector == NULL)?"":selector);
382+
383+ /* scan my cache, perhaps I have the content already */
384+ for (i = 0; i < TTLINIT; i++) {
385+ if (CACHE[i].id == NULL) continue;
386+ if (strcasecmp(CACHE[i].id, id) != 0) continue;
387+ /* found match! */
388+ printf("found id='%s' in CACHE[%d] (%ld bytes)\n", id, i, CACHE[i].bytes);
389+ if (CACHE[i].bytes > 0) memcpy(buff, CACHE[i].data, (unsigned long)(CACHE[i].bytes));
390+ if (CACHE[i].ipstr) strcpy(ipstr, CACHE[i].ipstr);
391+ return(CACHE[i].bytes);
392+ }
393+
394+ /* not found - fetch it for real */
395+ res = gopher_fetch(buff, buffsz, host, port, selector, ipstr, ipstrsz);
396+
397+ /* free old fields of the cache entry */
398+ free(CACHE[nextentry].id);
399+ free(CACHE[nextentry].ipstr);
400+ free(CACHE[nextentry].data);
401+ memset(&(CACHE[nextentry]), 0, sizeof(struct goph_cache));
402+
403+ /* write result to cache */
404+ CACHE[nextentry].id = strdup(id);
405+ CACHE[nextentry].ipstr = strdup(ipstr);
406+ if (res > 0) {
407+ CACHE[nextentry].data = malloc((unsigned long)res);
408+ memcpy(CACHE[nextentry].data, buff, (unsigned long)res);
409+ } else {
410+ CACHE[nextentry].data = NULL;
411+ }
412+ CACHE[nextentry].bytes = res;
413+
414+ /* */
415+ nextentry++;
416+ nextentry %= TTLINIT;
417+
418+ /* return */
419+ return(res);
420+}
421+
422+
423+static void mark_server_down(struct gopherlist **glist, const char *host, unsigned short port) {
424+ struct gopherlist *gnode;
425+
426+ gnode = glist_findhostport(*glist, host, port);
427+
428+ if (gnode == NULL) {
429+ printf("INTERNAL ERROR: tried to remove a server that is not found in glist ('%s' + port %u)\n", host, port);
430+ } else if (gnode->failedsince == 0) {
431+ /* server was working at some point in the past, but not anymore */
432+ gnode->failedsince = time(NULL);
433+ gnode->nextcheck = time(NULL) + CHECKPERIOD_FAST;
434+ printf("server %s:%u went down -> flaged as failed since now (%s)\n", gnode->fqdn, gnode->port, epoch2human(gnode->failedsince));
435+ } else if (time(NULL) > gnode->failedsince + MAXFAILTIME) {
436+ /* remove server from the list */
437+ printf("server removed due to long-time failure: %s:%u (failed since %s)\n", gnode->fqdn, gnode->port, epoch2human(gnode->failedsince));
438+ glist_node_unchain(glist, gnode);
439+ glist_node_free(&gnode);
440+ }
441+}
442+
443+
444+/* provides IPv4 address to scan when --scanipv4 enabled */
445+static void pick_an_ipv4_address_to_scan(struct gopherlist **glist) {
446+ static uint32_t lastip;
447+ static int firstrun = 1;
448+ char addr[16]; /* 255.255.255.255 is 15 bytes long (+1 for NULL terminator) */
449+ unsigned short runs = 10; /* add that many IP addresses */
450+
451+ /* if first time, start from a random position */
452+ if (firstrun) {
453+ firstrun = 0;
454+ lastip = (uint32_t)mrand48();
455+ }
456+
457+ while (runs-- != 0) {
458+ lastip++;
459+
460+ /* some ranges must be skipped */
461+ if ((lastip & 0xff000000lu) == 0) lastip = 0x01000000lu; /* 0.x.x.x -> 1.0.0.0 */
462+ if ((lastip & 0xff000000lu) == 0x0A000000lu) lastip = 0x0B000000lu; /* 10.x.x.x -> 11.0.0.0 */
463+ if ((lastip & 0xff000000lu) == 0x7F000000lu) lastip = 0x80000000lu; /* 127.x.x.x -> 128.0.0.0 */
464+ if ((lastip & 0xfff00000lu) == 0xAC100000lu) lastip = 0xAC200000lu; /* 172.[16-31].x.x -> 172.32.0.0 */
465+ if ((lastip & 0xffff0000lu) == 0xC0A80000lu) lastip = 0xC0A90000lu; /* 192.168.x.x -> 192.169.0.0 */
466+ if ((lastip >> 24) >= 224) lastip = 0x01000000lu; /* 224.0.0.0 - 255.255.255.255 -> 1.0.0.0 */
467+
468+ /* generate IP addr string */
469+ sprintf(addr, "%u.%u.%u.%u", (lastip >> 24) & 0xff, (lastip >> 16) & 0xff, (lastip >> 8) & 0xff, lastip & 0xff);
470+ printf("ipv4scan picked ip %s\n", addr);
471+
472+ /* insert glist entry */
473+ glist_addnewhostport(glist, addr, 70, 1);
474+ }
475+}
476+
477+
478+/**************** MAIN ****************/
479+
480+int main(int argc, char **argv) {
481+ time_t nextaction = 0;
482+ time_t nextdbsave;
483+ struct gopherlist *glist, *mlist, *gnode;
484+ struct gopherlist *curhost = NULL;
485+ char curhost_ipaddr[64];
486+ int ttl = 0;
487+ char buff[0xffff];
488+ long bufflen;
489+ char *dbfile, *dbfilecnt;
490+
491+ if (parseargs(argc, argv, &dbfile, &dbfilecnt) != 0) {
492+ printf("usage: gopherjoker [options] dbfile.csv dbcount.csv\n"
493+ "\n"
494+ "options:\n"
495+ "--waitperiod p sets delay period between spidering actions to p seconds (default=%d)\n"
496+ "--saveperiod p sets save frequency of the database file to p seconds (default=%d)\n"
497+ "--scanipv4 performs IPv4 address space scanning when there is nothing else to do\n"
498+ "\n",
499+ WAITPERIOD_DEFAULT,
500+ SAVEPERIOD_DEFAULT
501+ );
502+ return(1);
503+ }
504+
505+ nextdbsave = time(NULL) + SAVEPERIOD;
506+
507+ /* load db file */
508+ glist = loaddb(dbfile);
509+
510+ /* init random engine */
511+ srand48((long)time(NULL));
512+
513+ for (;;) {
514+
515+ /* make sure that my pending logs are output to stdout (avoids lags when redirecting to file) */
516+ fflush(stdout);
517+
518+ /* do not browse too fast */
519+ while (time(NULL) < nextaction) sleep(1);
520+ nextaction = time(NULL) + WAITPERIOD;
521+
522+ printf("\n\n\n--- [%s] ---\n", epoch2human(time(NULL)));
523+
524+ /* if extra manual hosts required to be added, do it now */
525+ loadextrahosts(&glist, NEWHOSTSDIR);
526+
527+ /* save the db once every hour */
528+ if (time(NULL) > nextdbsave) {
529+ unsigned long savedbcount;
530+ printf("dumping hosts lists to %s\n", dbfile);
531+ savedbcount = savedb(dbfile, dbfilecnt, glist);
532+ if (savedbcount == 0) printf("ERR: savedbcount == 0 @ %d\n", __LINE__);
533+ nextdbsave = time(NULL) + SAVEPERIOD;
534+ }
535+
536+ /* if ttl expired, pick a new host to browse */
537+ if ((--ttl == 0) || (curhost == NULL)) {
538+ printf("picking a new host to process...\n");
539+ curhost = pickhostfromlist(glist);
540+ if (curhost == NULL) {
541+ printf("no hosts to process. you can add some through " NEWHOSTSDIR ".\n");
542+ if (SCANIPV4SPACE != 0) {
543+ pick_an_ipv4_address_to_scan(&glist);
544+ } else {
545+ sleep(NOJOB_SLEEP);
546+ }
547+ continue;
548+ }
549+ curhost->nextcheck = time(NULL) + CHECKPERIOD;
550+ curhost = glist_node_dup(curhost);
551+ ttl = TTLINIT;
552+ }
553+
554+ printf("[TTL=%d] fetching %s:%u/1%s ...\n", ttl, curhost->fqdn, curhost->port, (curhost->selector == NULL)?"":curhost->selector);
555+
556+ /* fetch selector */
557+ bufflen = cached_gopher_fetch(buff, sizeof(buff), curhost->fqdn, curhost->port, curhost->selector, curhost_ipaddr, sizeof(curhost_ipaddr));
558+ if (bufflen < 1) { /* fail */
559+ printf("failed\n");
560+ /* if it was about root selector, see if it's time to drop the server */
561+ if (curhost->selector == NULL) mark_server_down(&glist, curhost->fqdn, curhost->port);
562+ glist_node_free(&curhost);
563+ continue;
564+ }
565+
566+ printf("ok (%ld bytes, IP=%s)\n", bufflen, curhost_ipaddr);
567+
568+ /* menu to glist */
569+ mlist = menu2gopherlist(buff, bufflen);
570+ if (mlist == NULL) {
571+ printf("ERR: no entries found in menu\n");
572+ /* if it was about root selector, consider lack of entries as a down state */
573+ if (curhost->selector == NULL) mark_server_down(&glist, curhost->fqdn, curhost->port);
574+ /* */
575+ glist_node_free(&curhost);
576+ continue;
577+ }
578+
579+ /* try adding hosts to global glist */
580+ for (gnode = mlist; gnode != NULL; gnode = gnode->next) {
581+ glist_addnewhostport(&glist, gnode->fqdn, gnode->port, 1);
582+ }
583+
584+ /* if main menu, then check that host is pointing to himself */
585+ if (curhost->selector == NULL) {
586+ /* make sure the server points to itself */
587+ if (glist_findhostport(mlist, curhost->fqdn, curhost->port) == NULL) {
588+ printf("ERR: main menu contains no link to self, dropping host:port '%s':%u\n", curhost->fqdn, curhost->port);
589+ gnode = glist_findhostport(glist, curhost->fqdn, curhost->port);
590+ glist_node_unchain(&glist, gnode);
591+ glist_node_free(&gnode);
592+ glist_node_free(&curhost);
593+ glist_free(mlist);
594+ continue;
595+ }
596+ }
597+
598+ /* mark host as 'okay' and update its IP address */
599+ gnode = glist_findhostport(glist, curhost->fqdn, curhost->port);
600+ gnode->failedsince = 0;
601+ strcpy(gnode->ipaddr, curhost_ipaddr);
602+
603+ /* curate the mlist by removing all entries with a NULL selector (these are not spider-able) */
604+ for (gnode = mlist; gnode != NULL; ) {
605+ struct gopherlist *g = gnode;
606+ gnode = gnode->next;
607+ if (g->selector != NULL) continue;
608+ /* drop the node */
609+ glist_node_unchain(&mlist, g);
610+ glist_node_free(&g);
611+ }
612+
613+ /* choose a random entry from mlist */
614+ gnode = pickrandhostfromlist(mlist);
615+ if (gnode == NULL) {
616+ printf("pickrandhostfromlist() could not find a node candidate in list\n");
617+ glist_node_free(&curhost);
618+ glist_free(mlist);
619+ continue;
620+ }
621+ curhost = glist_node_dup(gnode);
622+
623+ glist_free(mlist);
624+ }
625+ /* end of program */
626+}
--- tags/ogup-20221026/readme.txt (nonexistent)
+++ tags/ogup-20221026/readme.txt (revision 56)
@@ -0,0 +1,67 @@
1+
2+This directory contains the source files of the OGUP project.
3+
4+The web homepage of the ogup project is located at http://ogup.osdn.io
5+
6+
7+=== BUILD ====================================================================
8+
9+Building the OGUP requires an ANSI C compiler (preferably gcc or clang) to
10+build the 'gopherjoker' crawler.
11+
12+
13+=== RUN ======================================================================
14+
15+The 'frontend' directory can be served with a gopher server to present users
16+with the OGUP interface. This has been tested exclusively with the Motsognir
17+gopher server, and is unlikely to function properly with a different server.
18+
19+If the build process was successful, a 'gopherjoker' binary should have
20+appeared in the gopherjoker directory. This is a gopher crawler that will
21+spider the gopherspace. It should be executed as such:
22+
23+ $ gopherjoker ogupdb.dat ogupdb.cnt
24+
25+ogupdb.dat is a comma-separated-value text file that gopherjoker uses as its
26+database of known gopher servers. ogupdb.cnt is a file that contains only
27+global counters of known servers, that is used by the OGUP frontend.
28+
29+The ogupdb.dat and ogupdb.cnt files should be stored in the directory where
30+OGUP's frontend lives, so the frontend always has most up-to-date data.
31+
32+Both ogupdb.dat and ogupdb.cnt are updated only once an hour by gopherjoker,
33+to avoid too much filesystem updates.
34+
35+
36+=== LEARNING NEW SERVERS =====================================================
37+
38+gopherjoker learns about new gopher servers by discovering menu listings of
39+existing (known) servers. It is also possible to hand-feed gopherjoker with
40+new servers, the frontend has an appropriate submission form for that. This
41+form writes submissions to the /tmp/ogup/ directory, where they are picked up
42+by gopherjoker (and deleted by gopherjoker afterwards).
43+
44+
45+=== LICENSE ==================================================================
46+
47+All OGUP files are made available under the terms of the MIT License.
48+
49+Copyright (C) 2019-2021 Mateusz Viste
50+
51+Permission is hereby granted, free of charge, to any person obtaining a copy
52+of this software and associated documentation files (the "Software"), to deal
53+in the Software without restriction, including without limitation the rights
54+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
55+copies of the Software, and to permit persons to whom the Software is
56+furnished to do so, subject to the following conditions:
57+
58+The above copyright notice and this permission notice shall be included in all
59+copies or substantial portions of the Software.
60+
61+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
62+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
63+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
64+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
65+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
66+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
67+SOFTWARE.
Show on old repository browser