collecting prototype

collector.go

@@ -0,0 +1,33 @@
+// peertube-instance-index-filter
+// Copyright (C) 2025 Arns Udovič <zordsdavini@arns.lt>
+// 
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+// 
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+// 
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+type Instance struct {
+	Url string
+	Data string
+	CreatedAt string
+}
+
+func getNewHosts(url string, lastFetched string, start int, count int) ([]string, error) {
+	return []string{}, nil
+}
+
+func fetchInstance(url string, host string) Instance {
+	return Instance{}
+}
+
+

db.go

@@ -24,6 +24,7 @@ import (
 var migrations = []string{
 	"SELECT 1;",
 	"ALTER TABLE instances ADD COLUMN rejected INTEGER NOT NULL DEFAULT 0;",
+	"ALTER TABLE instances ADD COLUMN reject_reason TEXT;",
 }
 
 func connectDB() *sql.DB {
@@ -109,3 +110,28 @@ func addIndex(db *sql.DB, url string) {
 func rejectHost(db *sql.DB, host string) {
 	db.Exec("UPDATE instances SET rejected = 1 WHERE url = ?", host)
 }
+
+func getIndexUrls(db *sql.DB) map[string]string {
+	urls := make(map[string]string)
+	rows, err := db.Query("SELECT url, last_fetched_at FROM index_host")
+	if err != nil {
+		panic(err)
+	}
+
+	for rows.Next() {
+		var url string
+		var lastFetchedAt string
+		rows.Scan(&url, &lastFetchedAt)
+		urls[url] = lastFetchedAt
+	}
+
+	return urls
+}
+
+func updateLastFetched(db *sql.DB, url string) {
+	db.Exec("UPDATE index_host SET last_fetched_at = datetime('now') WHERE url = ?", url)
+}
+
+func addInstance(db *sql.DB, instance Instance) {
+	db.Exec("INSERT INTO instances (url, data, created_at) VALUES (?, ?, ?);", instance.Url, instance.Data, instance.CreatedAt)
+}

main.go

@@ -75,7 +75,35 @@ func reject(host string) {
 }
 
 func collect() {
-	// TODO: implement
+	db := connectDB()
+	defer db.Close()
+
+	urls := getIndexUrls(db)
+	for url, lastFetched := range urls {
+		fmt.Println(url)
+		
+		start := 0
+		count := 20
+		for {
+			hosts, err := getNewHosts(url, lastFetched, start, count)
+			if err != nil {
+				panic(err)
+			}
+
+			if len(hosts) == 0 {
+				break
+			}
+
+			for _, host := range hosts {
+				instance := fetchInstance(url, host)
+				addInstance(db, instance)
+			}
+
+			start += count
+		}
+
+		updateLastFetched(db, url)
+	}
 }
 
 func serve() {