// peertube-instance-index-filter // Copyright (C) 2025 Arns Udovič // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package main import ( "encoding/json" "flag" "fmt" "net/http" "net/url" "regexp" "strconv" "strings" "time" "github.com/gin-gonic/gin" ) type HostColection struct { Host string `json:"host"` } type HostsResponse struct { Total int `json:"total"` Data []HostColection `json:"data"` } type InstancesResponse struct { Total int `json:"total"` Data []json.RawMessage `json:"data"` } func main() { var command string var url string var instanceUrl string var host string var reason string flag.StringVar(&command, "command", "", "Command to execute: index, reject, collect, serve") flag.StringVar(&url, "url", "", "Url to index hosts") flag.StringVar(&instanceUrl, "instance-url", "", "Url to fetch instance information") flag.StringVar(&host, "host", "", "Host to reject") flag.StringVar(&reason, "reject-reason", "", "Reject reason (optional)") flag.Parse() fmt.Println(command, host) switch command { case "index": index(url, instanceUrl) case "reject": reject(host, reason) case "collect": collect() case "serve": default: serve() } } func index(url string, instanceUrl string) { db := connectDB() defer db.Close() exists, err := indexExists(db, url) if err != nil { panic(err) } if !exists { indexHost := IndexHost{ Url: url, InstanceUrl: instanceUrl, } addIndex(db, indexHost) fmt.Println(url, "added to index") } else { fmt.Println(url, "already added") } } func reject(host string, reason string) { host = formatHost(host) db := connectDB() defer db.Close() rejectHost(db, host, reason) fmt.Println(host, "rejected") } func collect() { db := connectDB() defer db.Close() indexHosts := getIndexHosts(db) for _, indexHost := range indexHosts { fmt.Println(indexHost.Url) fmt.Println("==========================================") start := 0 count := 20 for { hosts, err := getNewHosts(indexHost.Url, indexHost.LastFetchedAt, start, count) if err != nil { panic(err) } if len(hosts) == 0 { break } fmt.Println("New hosts:", len(hosts), hosts) for _, host := range hosts { fmt.Println(host) time.Sleep(1 * time.Second) instance := fetchInstance(indexHost.InstanceUrl, host) addInstance(db, instance) } start += count } updateLastFetched(db, indexHost) } } func serve() { db := connectDB() defer db.Close() r := gin.Default() r.GET("/", func(c *gin.Context) { c.String(200, "index") }) r.GET("/instances", func(c *gin.Context) { start := c.DefaultQuery("start", "0") count := c.DefaultQuery("count", "20") since := c.DefaultQuery("since", "") search := c.DefaultQuery("search", "") if since != "" { regex := regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`) if !regex.MatchString(since) { generateErrorResponse(c, "since must be in YYYY-MM-DD format") return } } starti, err := strconv.Atoi(start) if err != nil { generateErrorResponse(c, "start should be an integer") return } counti, err := strconv.Atoi(count) if err != nil { generateErrorResponse(c, "count should be an integer") return } hosts, err := getHosts(db, starti, counti, since, search, "data") if err != nil { generateErrorResponse(c, "error getting hosts") return } total, err := getHostsTotal(db, since, search) if err != nil { generateErrorResponse(c, "error getting hosts") return } response := InstancesResponse{ Total: total, } for _, data := range hosts { response.Data = append(response.Data, []byte(data)) } c.JSON(http.StatusOK, response) }) r.GET("/instances/hosts", func(c *gin.Context) { start := c.DefaultQuery("start", "0") count := c.DefaultQuery("count", "20") since := c.DefaultQuery("since", "") if since != "" { regex := regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`) if !regex.MatchString(since) { generateErrorResponse(c, "since must be in YYYY-MM-DD format") return } } starti, err := strconv.Atoi(start) if err != nil { generateErrorResponse(c, "start should be an integer") return } counti, err := strconv.Atoi(count) if err != nil { generateErrorResponse(c, "count should be an integer") return } hosts, err := getHosts(db, starti, counti, since, "", "url") if err != nil { generateErrorResponse(c, "error getting hosts") return } total, err := getHostsTotal(db, since, "") if err != nil { generateErrorResponse(c, "error getting hosts") return } response := HostsResponse{ Total: total, } for _, host := range hosts { response.Data = append(response.Data, HostColection{ Host: host, }) } c.JSON(http.StatusOK, response) }) r.Run(":8081") } func formatHost(host string) string { host = strings.Trim(host, " ") u, _ := url.Parse(host) if u.Host == "" { return host } return u.Host } func generateErrorResponse(c *gin.Context, err string) { c.JSON(http.StatusBadRequest, gin.H{"error": err}) }