peertube-instance-index-filter/main.go

// peertube-instance-index-filter
// Copyright (C) 2025 Arns Udovič <zordsdavini@arns.lt>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
)

type HostColection struct {
	Host  string `json:"host"`
}
type HostsResponse struct {
  Total   int `json:"total"`
  Data    []HostColection `json:"data"`
}

type InstancesResponse struct {
  Total   int `json:"total"`
  Data    []json.RawMessage `json:"data"`
}

func main() {
	var command string
	var url string
	var instanceUrl string
	var host string
	var reason string
	var filter string
	var lastId string

	flag.StringVar(&command, "command", "", "Command to execute: index, reject, collect, serve, audit")
	flag.StringVar(&url, "url", "", "Url to index hosts")
	flag.StringVar(&instanceUrl, "instance-url", "", "Url to fetch instance information")
	flag.StringVar(&host, "host", "", "Host to reject")
	flag.StringVar(&reason, "reject-reason", "", "Reject reason (optional)")
	flag.StringVar(&filter, "filter", "", "filter for audit")
	flag.StringVar(&lastId, "last-id", "", "last id")

	flag.Parse()
	fmt.Println(command, host)

	start, err := strconv.Atoi(lastId)
	if err != nil {
		start = 0
	}
	
	switch command {
		case "index":
			index(url, instanceUrl)
		case "reject":
			reject(host, reason)
		case "collect":
			collect()
		case "audit":
			audit(filter, start)
		case "serve":
		default:
			serve()
	}
}

func index(url string, instanceUrl string) {
	db := connectDB()
	defer db.Close()

	exists, err := indexExists(db, url)
	if err != nil {
		panic(err)	
	}

	if !exists {
		indexHost := IndexHost{
			Url: url,
			InstanceUrl: instanceUrl,
		}
		addIndex(db, indexHost)
		fmt.Println(url, "added to index")
	} else {
		fmt.Println(url, "already added")
	}
}

func reject(host string, reason string) {
	host = formatHost(host)

	db := connectDB()
	defer db.Close()

	rejectHost(db, host, reason)
	fmt.Println(host, "rejected")
}

func collect() {
	db := connectDB()
	defer db.Close()

	indexHosts := getIndexHosts(db)
	for _, indexHost := range indexHosts {
		fmt.Println(indexHost.Url)
		fmt.Println("==========================================")
		
		start := 0
		count := 20
		for {
			hosts, err := getNewHosts(indexHost.Url, indexHost.LastFetchedAt, start, count)
			if err != nil {
				panic(err)
			}

			if len(hosts) == 0 {
				break
			}

			fmt.Println("New hosts:", len(hosts), hosts)
			for _, host := range hosts {
				fmt.Println(host)

				time.Sleep(1 * time.Second)
				instance := fetchInstance(indexHost.InstanceUrl, host)
				addInstance(db, instance)
			}

			start += count
		}

		updateLastFetched(db, indexHost)
	}
}

func audit(filter string, start int) {
	db := connectDB()
	defer db.Close()

	if filter == "duplicates" {
		removeDuplicates()
		return
	}

	hosts, err := getHosts(db, start, 10000, "", "", "url")
	if err != nil {
		panic(err)
	}

	for _, url := range hosts {
		filtered := doAudit(url, filter)
		if filtered {
			reject(url, filter)
		}
	}
}

func serve() {
	db := connectDB()
	defer db.Close()

	r := gin.Default()
	r.LoadHTMLGlob("templates/*")

	r.GET("/", func(c *gin.Context) {
		filteredInstancesCount, err := getHostsTotal(db, "", "")
		if err != nil {
			panic(err)			
		}

		rejectedInstancesCount, err := getRejectedHostsTotal(db)
		if err != nil {
			panic(err)			
		}

		c.HTML(http.StatusOK, "index.tmpl", gin.H{
			"FilteredInstancesCount": filteredInstancesCount,
			"RejectedInstancesCount": rejectedInstancesCount,
		})
	})

	r.GET("/instances", func(c *gin.Context) {
		start := c.DefaultQuery("start", "0")
		count := c.DefaultQuery("count", "20")
		since := c.DefaultQuery("since", "")
		search := c.DefaultQuery("search", "")

		if since != "" {
			regex := regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`)
			if !regex.MatchString(since) {
				generateErrorResponse(c, "since must be in YYYY-MM-DD format")
				return
			}
		}

		starti, err := strconv.Atoi(start)
		if err != nil {
			generateErrorResponse(c, "start should be an integer")
			return
		}

		counti, err := strconv.Atoi(count)
		if err != nil {
			generateErrorResponse(c, "count should be an integer")
			return
		}

		hosts, err := getHosts(db, starti, counti, since, search, "data")
		if err != nil {
			generateErrorResponse(c, "error getting hosts")
			return
		}

		total, err := getHostsTotal(db, since, search)
		if err != nil {
			generateErrorResponse(c, "error getting hosts")
			return
		}

		response := InstancesResponse{
			Total: total,
		}

		for _, data := range hosts {
			response.Data = append(response.Data, []byte(data))
		}

		c.JSON(http.StatusOK, response)
	})

	r.GET("/instances/hosts", func(c *gin.Context) {
		start := c.DefaultQuery("start", "0")
		count := c.DefaultQuery("count", "20")
		since := c.DefaultQuery("since", "")

		if since != "" {
			regex := regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`)
			if !regex.MatchString(since) {
				generateErrorResponse(c, "since must be in YYYY-MM-DD format")
				return
			}
		}

		starti, err := strconv.Atoi(start)
		if err != nil {
			generateErrorResponse(c, "start should be an integer")
			return
		}

		counti, err := strconv.Atoi(count)
		if err != nil {
			generateErrorResponse(c, "count should be an integer")
			return
		}

		hosts, err := getHosts(db, starti, counti, since, "", "url")
		if err != nil {
			generateErrorResponse(c, "error getting hosts")
			return
		}

		total, err := getHostsTotal(db, since, "")
		if err != nil {
			generateErrorResponse(c, "error getting hosts")
			return
		}

		response := HostsResponse{
			Total: total,
		}

		for _, host := range hosts {
			response.Data = append(response.Data, HostColection{
				Host: host,
			})
		}

		c.JSON(http.StatusOK, response)
	})

	r.Run(":8081")
}

func formatHost(host string) string {
	host = strings.Trim(host, " ")

	u, _ := url.Parse(host)

	if u.Host == "" {
		return host
	}

	return u.Host
}

func generateErrorResponse(c *gin.Context, err string) {
	c.JSON(http.StatusBadRequest, gin.H{"error": err})
}