Select Git revision
server_apps.go

Timon authored
server_apps.go 22.00 KiB
package main
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"context"
"crypto/md5"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/url"
"path"
"regexp"
"strings"
"strconv"
"time"
"math"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/storage/memory"
"github.com/pelletier/go-toml/v2"
)
type appStyle struct {
Background string `json:"background",omitempty`
Color string `json:"color",omitempty`
}
type appPatch struct{
version int
repository string
zip []byte
targz []byte
}
type appStatus struct{
tested_version int
broken bool
patch *appPatch
}
type appRegistry struct {
shame []*appShame
apps []*appDescriptor
}
type appShame struct {
repository string
errorMsg string
}
type appDescriptor struct {
repository string
pathInRepo string
appInfo *appInfo
}
type appInfo struct {
name string
menu string
author string
description string
version int
slug string
commit string
stars int
flow3rSeed string
commitObj *object.Commit
zip []byte
targz []byte
firstErr error
featured bool
status *appStatus
style *appStyle
}
type GLProject struct {
StarCount int `json:"star_count"`
}
var (
reAppPath = regexp.MustCompile(`^apps/([^/]+.toml)$`)
reAppRepo = regexp.MustCompile(`^([a-zA-Z\-_\.0-9]+)/([a-zA-Z\-_0-9]+)$`)
)
func (s *server) getStars(ctx context.Context, repo string) (int, error) {
path := fmt.Sprintf("https://%s/api/v4/projects/%s", flagGitlabHost, url.PathEscape(repo))
req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
if err != nil {
return 0, fmt.Errorf("when building request: %w", err)
}
res, err := http.DefaultTransport.RoundTrip(req)
if err != nil {
return 0, fmt.Errorf("when performing request: %w", err)
}
defer res.Body.Close()
if res.StatusCode != 200 {
return 0, fmt.Errorf("invalid response code %d", res.StatusCode)
}
var project GLProject
j := json.NewDecoder(res.Body)
err = j.Decode(&project)
if err != nil {
return 0, fmt.Errorf("when performing request: %w", err)
}
return project.StarCount, nil
}
func (s *server) zipApp(ctx context.Context, name, pathInRepo, repo string, patch_source *string, obj *object.Commit) ([]byte, error) {
fi, err := obj.Files()
if err != nil {
return nil, fmt.Errorf("listing files: %w", err)
}
buf := bytes.NewBuffer(nil)
w := zip.NewWriter(buf)
for {
f, err := fi.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("fi.Next: %w", err)
}
prefix := ""
if pathInRepo != "" {
prefix = pathInRepo + "/"
}
if !strings.HasPrefix(f.Name, prefix) {
continue
}
if !f.Mode.IsFile() {
continue
}
p := f.Name[len(prefix):]
prefix = strings.ReplaceAll(repo, "/", "-")
outPath := path.Join(prefix, p)
fo, err := w.Create(outPath)
if err != nil {
return nil, fmt.Errorf("Create(%q): %w", outPath, err)
}
if f.Size+int64(buf.Len()) > 10<<20 {
return nil, fmt.Errorf("archive too large")
}
rdr, err := f.Blob.Reader()
if err != nil {
return nil, fmt.Errorf("Blob.Reader: %w", err)
}
if f.Name == "flow3r.toml" && patch_source != nil{
fo.Write([]byte(fmt.Sprintf("patch_source = \"%s\"\n\n", *patch_source)))
}
_, err = io.Copy(fo, rdr)
rdr.Close()
if err != nil {
return nil, fmt.Errorf("when copying: %w", err)
}
}
if err := w.Close(); err != nil {
return nil, fmt.Errorf("when closing: %w", err)
}
return buf.Bytes(), nil
}
func (s *server) targzApp(ctx context.Context, name, pathInRepo, repo string, patch_source * string, obj *object.Commit) ([]byte, error) {
fi, err := obj.Files()
if err != nil {
return nil, fmt.Errorf("listing files: %w", err)
}
buf := bytes.NewBuffer(nil)
gz := gzip.NewWriter(buf)
t := tar.NewWriter(gz)
directories := make(map[string]bool)
for {
f, err := fi.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("fi.Next: %w", err)
}
prefix := ""
if pathInRepo != "" {
prefix = pathInRepo + "/"
}
if !strings.HasPrefix(f.Name, prefix) {
continue
}
if !f.Mode.IsFile() {
continue
}
p := f.Name[len(prefix):]
prefix = strings.ReplaceAll(repo, "/", "-")
outPath := path.Join(prefix, p)
ensureDir := func(p string) error {
if directories[p] {
return nil
}
directories[p] = true
err = t.WriteHeader(&tar.Header{
Name: p,
Typeflag: tar.TypeDir,
Mode: 0755,
})
if err != nil {
return fmt.Errorf("CreateDir(%q): %w", p, err)
}
return nil
}
dirs, _ := path.Split(outPath)
if dirs != "" {
dirs = strings.TrimRight(dirs, "/")
parts := strings.Split(dirs, "/")
cur := parts[0] + "/"
if err := ensureDir(cur); err != nil {
return nil, err
}
for i := 1; i < len(parts); i++ {
cur += parts[i] + "/"
if err := ensureDir(cur); err != nil {
return nil, err
}
}
}
patch_bytes := []byte("")
if f.Name == "flow3r.toml" && patch_source != nil{
patch_bytes = []byte(fmt.Sprintf("patch_source = \"%s\"\n\n", *patch_source))
}
err = t.WriteHeader(&tar.Header{
Name: outPath,
Typeflag: tar.TypeReg,
Size: f.Size + int64(len(patch_bytes)),
Mode: 0644,
})
if err != nil {
return nil, fmt.Errorf("Create(%q): %w", outPath, err)
}
if f.Size+int64(buf.Len()) > 10<<20 {
return nil, fmt.Errorf("archive too large")
}
rdr, err := f.Blob.Reader()
if err != nil {
return nil, fmt.Errorf("Blob.Reader: %w", err)
}
if len(patch_bytes) != 0{
t.Write(patch_bytes)
}
if err != nil {
return nil, fmt.Errorf("patch_source: %w", err)
}
_, err = io.Copy(t, rdr)
rdr.Close()
if err != nil {
return nil, fmt.Errorf("when copying: %w", err)
}
}
if err := t.Close(); err != nil {
return nil, fmt.Errorf("when closing tar: %w", err)
}
if err := gz.Close(); err != nil {
return nil, fmt.Errorf("when closing gz: %w", err)
}
return buf.Bytes(), nil
}
func (s *server) parseAppToml(ctx context.Context, pathInRepo string, obj *object.Commit) (*appInfo, error) {
p := path.Join(pathInRepo, "flow3r.toml")
f, err := obj.File(p)
if err != nil {
return nil, fmt.Errorf("toml open (%q) failed: %w", p, err)
}
reader, err := f.Reader()
if err != nil {
return nil, fmt.Errorf("f.Reader: %w", err)
}
defer reader.Close()
var data struct {
App struct {
Name string
Category string
Menu string
}
Entry struct {
Class string
}
Metadata struct {
Author string
License string
URL string
Description string
Version int
}
Style map[string]interface{}
}
dec := toml.NewDecoder(reader)
err = dec.Decode(&data)
if err != nil {
return nil, fmt.Errorf("toml decode (%q) failed: %w", p, err)
}
if data.App.Name == "" || len(data.App.Name) > 32 {
return nil, fmt.Errorf("app name invalid (must be non-empty and <= 32 chars)")
}
if data.App.Category == "" {
data.App.Category = data.App.Menu
}
sections := map[string]bool{
"Badge": true,
"Apps": true,
"Music": true,
"Media": true,
"Games": true,
"Demos": true,
}
if !sections[data.App.Category] {
return nil, fmt.Errorf("app category invalid (must be one of 'Badge', 'Apps', 'Music', 'Media', 'Games', 'Demos')")
}
if data.Entry.Class == "" {
return nil, fmt.Errorf("no entry class")
}
if len(data.Metadata.Author) > 32 {
return nil, fmt.Errorf("metadata author too long (must be <= 32 chars)")
}
if len(data.Metadata.Description) > 140 {
return nil, fmt.Errorf("metadata description too long (must be <= 140 chars)")
}
ret := appInfo{
name: data.App.Name,
author: data.Metadata.Author,
menu: data.App.Category,
description: data.Metadata.Description,
version: data.Metadata.Version,
commit: obj.Hash.String(),
commitObj: obj,
style: nil,
}
style := appStyle{}
style_exists := false
if bg_col, ok := data.Style["background"].(string); ok {
style.Background = bg_col
style_exists = true
}
if col, ok := data.Style["color"].(string); ok {
style.Color = col
style_exists = true
}
if style_exists {
ret.style = &style
}
return &ret, nil
}
func (s *server) getAppInfo(ctx context.Context, pathInRepo, repo string, slug *string) (*appInfo, error) {
url := "https://git.flow3r.garden/" + repo
g, err := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
URL: url,
})
if err != nil {
return nil, fmt.Errorf("when cloning: %w", err)
}
cfg, err := g.Config()
if err != nil {
return nil, fmt.Errorf("when getting config: %w", err)
}
if len(cfg.Branches) < 1 {
return nil, fmt.Errorf("no branches")
}
var bname string
for name := range cfg.Branches {
bname = name
break
}
h, err := g.ResolveRevision(plumbing.Revision(bname))
if err != nil {
return nil, fmt.Errorf("resolving revision failed: %w", err)
}
err = g.Fetch(&git.FetchOptions{
RefSpecs: []config.RefSpec{
config.RefSpec(fmt.Sprintf("%s:refs/heads/%s", bname, bname)),
},
})
if err != nil && err != git.NoErrAlreadyUpToDate {
return nil, fmt.Errorf("fetch failed: %w", err)
}
obj, err := g.CommitObject(*h)
if err != nil {
return nil, fmt.Errorf("CommitObject(%v): %w", h, err)
}
highestVer := 0
highsetVerNil := true
firstTime := make(map[int]*appInfo)
var firstErr error = nil
for {
info, err := s.parseAppToml(ctx, pathInRepo, obj)
if err == nil {
ver := info.version
firstTime[ver] = info
if ver > highestVer || highsetVerNil {
highestVer = ver
highsetVerNil = false
}
} else {
if highsetVerNil && firstErr == nil {
firstErr = err
log.Printf("%s@%s: latest error: %v", repo, obj.Hash.String(), err)
} else {
log.Printf("%s@%s: %v", repo, obj.Hash.String(), err)
}
}
// just get the latest commit
break
if len(obj.ParentHashes) == 0 {
break
}
obj, err = g.CommitObject(obj.ParentHashes[0])
if err != nil {
return nil, fmt.Errorf("CommitObject(%v): %w", h, err)
}
}
if highsetVerNil {
if firstErr != nil {
return nil, firstErr
}
return nil, fmt.Errorf("no `version` field in `flow3r.toml`")
}
stars, err := s.getStars(ctx, repo)
if err != nil {
return nil, fmt.Errorf("getting gitlab stars failed: %w", err)
}
var patch_source *string = nil
slug_str := repo
if slug != nil{
slug_str = *slug
psrc := fmt.Sprintf("https://git.flow3r.garden/%s", repo)
patch_source = &psrc
}
app := firstTime[highestVer]
app.stars = stars
zbytes, err := s.zipApp(ctx, app.name, pathInRepo, slug_str, patch_source, app.commitObj)
if err != nil {
return nil, fmt.Errorf("zipping failed: %w", err)
}
app.zip = zbytes
tbytes, err := s.targzApp(ctx, app.name, pathInRepo, slug_str, patch_source, app.commitObj)
if err != nil {
return nil, fmt.Errorf("targzing failed: %w", err)
}
app.targz = tbytes
app.slug = strings.ReplaceAll(slug_str, "/", "-")
// Calculate an 8 digit flow3r seed which can be used to install the
// app. This is based on md5 so ambitious hack3rs can force a certain
// app seed :)
flow3rSeedMd5 := md5.Sum([]byte(repo))
app.flow3rSeed = ""
for i := 0; i < 8; i++ {
app.flow3rSeed += string(flow3rSeedMd5[i]%5 + byte('0'))
}
app.firstErr = firstErr
return app, nil
}
func getFlags(obj *object.Commit) (*map[string]interface{}, error){
flags, err := obj.File("tags.json")
if err != nil {
return nil, err
}
flagreader, err := flags.Reader()
if err != nil {
return nil, err
}
flagbytes, err := io.ReadAll(flagreader)
if err != nil {
return nil, err
}
var flagmap map[string]interface{}
err = json.Unmarshal([]byte(flagbytes), &flagmap)
if err != nil {
return nil, err
}
return &flagmap, nil
}
func (s *server) getAppRegistry(ctx context.Context) (*appRegistry, error) {
s.gitMu.Lock()
defer s.gitMu.Unlock()
if s.appRepo == nil {
git, err := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
URL: "https://git.flow3r.garden/flow3r/flow3r-apps",
})
if err != nil {
return nil, err
}
s.appRepo = git
}
if time.Since(s.lastFetch) > 10*time.Second {
s.appRepo.Fetch(&git.FetchOptions{
RefSpecs: []config.RefSpec{
config.RefSpec("+refs/heads/*:refs/heads/*"),
},
Force: true,
})
}
h, err := s.appRepo.ResolveRevision(plumbing.Revision("main"))
if err != nil {
return nil, err
}
obj, err := s.appRepo.CommitObject(*h)
if err != nil {
return nil, err
}
iter, err := obj.Files()
if err != nil {
return nil, err
}
flagmap, err := getFlags(obj)
if err != nil {
log.Printf("flags.json: couldn't read: %v", err)
}
var registry appRegistry
for {
f, err := iter.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
if !f.Mode.IsFile() {
continue
}
m := reAppPath.FindStringSubmatch(f.Name)
if m == nil {
continue
}
reader, err := f.Reader()
if err != nil {
log.Printf("App %q: couldn't read TOML: %v", m[1], err)
registry.shame = append(registry.shame, &appShame{
repository: fmt.Sprintf("<app-list toml: %q>", m[1]),
errorMsg: fmt.Sprintf("couldn't read TOML: %v", err),
})
continue
}
dec := toml.NewDecoder(reader)
var dat struct {
Repo string
PathInRepo string `toml:"path_in_repo"`
}
err = dec.Decode(&dat)
reader.Close()
if err != nil {
log.Printf("App %q: couldn't parse TOML: %v", m[1], err)
registry.shame = append(registry.shame, &appShame{
repository: fmt.Sprintf("<app-list toml: %q>", m[1]),
errorMsg: fmt.Sprintf("couldn't read TOML: %v", err),
})
continue
}
n := reAppRepo.FindStringSubmatch(dat.Repo)
if n == nil {
log.Printf("App %q: invalid repo %q", m[1], dat.Repo)
registry.shame = append(registry.shame, &appShame{
repository: fmt.Sprintf("<app-list toml: %q>", m[1]),
errorMsg: fmt.Sprintf("invalid repo %q", dat.Repo),
})
continue
}
registry.apps = append(registry.apps, &appDescriptor{
repository: dat.Repo,
pathInRepo: dat.PathInRepo,
})
}
for _, app := range registry.apps {
printed := false
info, err := s.getAppInfo(ctx, app.pathInRepo, app.repository, nil)
if err != nil {
printed = true
log.Printf("App %q: %v", app.repository, err)
registry.shame = append(registry.shame, &appShame{
repository: app.repository,
errorMsg: fmt.Sprintf("%v", err),
})
continue
} else if info.firstErr != nil {
printed = true
log.Printf("App: %q: was okay, latest has error: %v", app.repository, info.firstErr)
registry.shame = append(registry.shame, &appShame{
repository: app.repository,
errorMsg: fmt.Sprintf("%v", info.firstErr),
})
} else if flagmap != nil {
if app_flags, ok := (*flagmap)[app.repository].(map[string]interface{}); ok{
if feat, ok := app_flags["featured"].(bool); ok {
info.featured = feat
}
if status, ok := app_flags["status"].(map[string]interface{}); ok {
info.status = new(appStatus)
if tested, ok := status["tested_version"].(float64); ok {
info.status.tested_version = int(math.Round(tested))
}
if broken, ok := status["broken"].(bool); ok {
info.status.broken = broken
}
if patch, ok := status["patch"].(string); ok {
printed = true
patch_info, err := s.getAppInfo(ctx, "", patch, &app.repository)
if err != nil {
log.Printf("App %q: couldn't get patch at %q: %v", app.repository, patch, err)
} else {
log.Printf("App: %q: okay, patch found", app.repository)
info.status.patch = &appPatch{
repository: patch,
zip: patch_info.zip,
targz: patch_info.targz,
version: patch_info.version,
}
}
}
}
}
}
if !printed{
log.Printf("App: %q: okay", app.repository)
}
app.appInfo = info
}
log.Printf("Got %d apps and %d apps with shame", len(registry.apps), len(registry.shame))
return ®istry, nil
}
type jsonAppPatch struct{
Version int `json:"version"`
RepoURL string `json:"repoUrl"`
DownloadURL string `json:"downloadUrl"`
TarDownloadURL string `json:"tarDownloadUrl"`
}
type jsonAppStatus struct{
TestedVersion int `json:"tested_version"`
Broken bool `json:"broken"`
Patch *jsonAppPatch `json:"patch",omitempty`
}
type jsonApp struct {
RepoURL string `json:"repoUrl"`
Commit string `json:"commit"`
DownloadURL string `json:"downloadUrl"`
TarDownloadURL string `json:"tarDownloadUrl"`
Name string `json:"name"`
Menu string `json:"menu"`
Author string `json:"author"`
Description string `json:"description"`
Version int `json:"version"`
Stars int `json:"stars"`
Slug string `json:"slug"`
Featured bool `json:"featured"`
Timestamp time.Time `json:"timestamp"`
Flow3rSeed string `json:"flow3rSeed"`
Status *jsonAppStatus `json:"status",omitempty`
Style *appStyle `json:"style",omitempty`
}
func makeJsonApp(a *appDescriptor) jsonApp {
ret := jsonApp{
RepoURL: "https://git.flow3r.garden/" + a.repository,
Commit: a.appInfo.commit,
DownloadURL: fmt.Sprintf("%sapps/zip/%s.zip", flagBaseURL, a.repository),
TarDownloadURL: fmt.Sprintf("%sapps/tar/%s.tar.gz", flagBaseURL, a.repository),
Name: a.appInfo.name,
Menu: a.appInfo.menu,
Author: a.appInfo.author,
Description: a.appInfo.description,
Version: a.appInfo.version,
Timestamp: a.appInfo.commitObj.Committer.When.UTC(),
Stars: a.appInfo.stars,
Flow3rSeed: a.appInfo.flow3rSeed,
Featured: a.appInfo.featured,
Slug: a.appInfo.slug,
Style: a.appInfo.style,
}
if a.appInfo.status != nil{
ret.Status = &jsonAppStatus{
Broken: a.appInfo.status.broken,
TestedVersion: a.appInfo.status.tested_version,
}
if a.appInfo.status.patch != nil{
pa := a.appInfo.status.patch.repository
ret.Status.Patch = &jsonAppPatch{
RepoURL: "https://git.flow3r.garden/" + pa,
DownloadURL: fmt.Sprintf("%sapps/zip/%s.zip", flagBaseURL, pa),
TarDownloadURL: fmt.Sprintf("%sapps/tar/%s.tar.gz", flagBaseURL, pa),
Version: a.appInfo.status.patch.version,
}
}
}
return ret
}
func (s *server) handleApps(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
apps, err := s.getApps(ctx)
if err != nil {
return
}
type res struct {
DT time.Time `json:"isodatetime"`
Apps []jsonApp `json:"apps"`
}
resp := res{
DT: time.Now().UTC(),
}
for _, a := range apps {
if a.appInfo == nil {
continue
}
// Guarenteed to be exactly 2 parts because of the regex
resp.Apps = append(resp.Apps, makeJsonApp(a))
}
w.Header().Add("Access-Control-Allow-Origin", "*")
w.Header().Add("Content-Type", "application/json")
j := json.NewEncoder(w)
j.Encode(resp)
}
var (
reAppURL = regexp.MustCompile("^/api/apps/([0-4]{8}).json$")
)
func (s *server) handleApp(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
matches := reAppURL.FindStringSubmatch(r.URL.Path)
if matches == nil {
http.NotFound(w, r)
return
}
flow3rSeed := matches[1]
apps, err := s.getApps(ctx)
if err != nil {
return
}
for _, a := range apps {
if a.appInfo == nil {
continue
}
if a.appInfo.flow3rSeed == flow3rSeed {
w.Header().Add("Access-Control-Allow-Origin", "*")
w.Header().Add("Content-Type", "application/json")
j := json.NewEncoder(w)
j.Encode(makeJsonApp(a))
return
}
}
http.NotFound(w, r)
}
func (s *server) handleAppShame(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
appShame, err := s.getAppShame(ctx)
if err != nil {
return
}
type jsonAppShame struct {
Repo string `json:"repo"`
ErrorMsg string `json:"errorMsg"`
}
type res struct {
DT time.Time `json:"isodatetime"`
Apps []jsonAppShame `json:"shame"`
}
resp := res{
DT: time.Now().UTC(),
}
for _, shame := range appShame {
// Guarenteed to be exactly 2 parts because of the regex
resp.Apps = append(resp.Apps, jsonAppShame{
Repo: shame.repository,
ErrorMsg: shame.errorMsg,
})
}
w.Header().Add("Access-Control-Allow-Origin", "*")
w.Header().Add("Content-Type", "application/json")
j := json.NewEncoder(w)
j.Encode(resp)
}
var (
reAppZipURL = regexp.MustCompile(`^/api/apps/zip/([^/]+)/([^/]+)\.zip$`)
reAppTargzURL = regexp.MustCompile(`^/api/apps/tar/([^/]+)/([^/]+)\.tar\.gz$`)
)
func (s *server) handleAppZip(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
matches := reAppZipURL.FindStringSubmatch(r.URL.Path)
if matches == nil {
http.NotFound(w, r)
return
}
orga := matches[1]
repo := matches[2]
apps, err := s.getApps(ctx)
if err != nil {
return
}
repository := fmt.Sprintf("%s/%s", orga, repo)
for _, app := range apps {
if app.repository == repository {
w.Header().Add("Content-Type", "application/zip")
w.Write(app.appInfo.zip)
return
}
if app.appInfo == nil || app.appInfo.status == nil || app.appInfo.status.patch == nil {
continue
}
if app.appInfo.status.patch.repository == repository {
w.Header().Add("Content-Type", "application/zip")
w.Write(app.appInfo.status.patch.zip)
return
}
}
http.NotFound(w, r)
}
func (s *server) handleAppTargz(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
matches := reAppTargzURL.FindStringSubmatch(r.URL.Path)
if matches == nil {
http.NotFound(w, r)
return
}
orga := matches[1]
repo := matches[2]
repository := fmt.Sprintf("%s/%s", orga, repo)
apps, err := s.getApps(ctx)
if err != nil {
return
}
for _, app := range apps {
if app.repository == repository {
w.Header().Add("Content-Type", "application/x-gzip")
w.Header().Add("Content-Length", strconv.Itoa(len(app.appInfo.targz)))
w.Write(app.appInfo.targz)
return
}
if app.appInfo == nil || app.appInfo.status == nil || app.appInfo.status.patch == nil {
continue
}
if app.appInfo.status.patch.repository == repository {
w.Header().Add("Content-Type", "application/x-gzip")
w.Header().Add("Content-Length", strconv.Itoa(len(app.appInfo.status.patch.targz)))
w.Write(app.appInfo.status.patch.targz)
return
}
}
http.NotFound(w, r)
}