diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ed334b4eeed11ef7a440231ba906deca6c4c8020 --- /dev/null +++ b/README.md @@ -0,0 +1,28 @@ +flow3r api +========== + +A little monolithic API service for accessing stuff about the flow3r badge, +especially release information. + +Running +------- + + $ go run . + +APIs +---- + +| Endpoint | Description | +|-------------------------------------|---------------------------------------------------------| +| `/releases.json` | WebFlasher compatible list of releases | +| `/release/<version>/<artifact.bin>` | Artifacts from releases, pointed to by `/releases.json` | + +Internals +--------- + +`releases.json` is generated on demand from a cached view from the GitLab API. +Currently up to 50 releases are supported, as pagionation hasn't yet been +implemented. + +All artifacts are cached in memory after being extracted from tarballs. Might +need to be offloaded to some other storage. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..3bfb5319f8b0bda2576725f09994e3b5c1e9e3a8 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module git.flow3r.garden/flowe3r/api + +go 1.20 diff --git a/main.go b/main.go new file mode 100644 index 0000000000000000000000000000000000000000..e3e72058d9f61db0d3b3163a68b6d7de11c61800 --- /dev/null +++ b/main.go @@ -0,0 +1,292 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/bzip2" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "net/url" + "regexp" + "strings" + "sync" + "time" +) + +var ( + flagBaseURL string + flagListen string + flagGitlabHost string + flagGitlabProject string +) + +// https://git.flow3r.garden/api/v4/projects/flow3r%2Fflow3r-firmware/releases + +type GLAssetLink struct { + ID int64 `json:"id"` + Name string `json:"name"` + TagName string `json:"tag_name"` + URL string `json:"url"` +} + +type GLRelease struct { + Name string `json:"name"` + TagName string `json:"tag_name"` + ReleasedAt time.Time `json:"released_at"` + Assets struct { + Links []GLAssetLink `json:"links"` + } `json:"assets"` +} + +func getReleases(ctx context.Context) ([]GLRelease, error) { + path := fmt.Sprintf("https://%s/api/v4/projects/%s/releases?order_by=created_at", flagGitlabHost, url.PathEscape(flagGitlabProject)) + req, err := http.NewRequestWithContext(ctx, "GET", path, nil) + if err != nil { + return nil, fmt.Errorf("when building request: %w", err) + } + res, err := http.DefaultTransport.RoundTrip(req) + if err != nil { + return nil, fmt.Errorf("when performing request: %w", err) + } + defer res.Body.Close() + + if res.StatusCode != 200 { + return nil, fmt.Errorf("invalid response code %d", res.StatusCode) + } + + var releases []GLRelease + j := json.NewDecoder(res.Body) + err = j.Decode(&releases) + if err != nil { + return nil, fmt.Errorf("when performing request: %w", err) + } + return releases, nil +} + +type req struct { + getReleases chan []GLRelease +} + +type server struct { + reqC chan *req + + cacheMu sync.RWMutex + cache map[string]map[string][]byte +} + +func (s *server) run(ctx context.Context) { + releases, err := getReleases(ctx) + if err != nil { + log.Fatalf("Initial release fetch failed: %v", err) + } + + t := time.NewTicker(60 * time.Second) + for { + select { + case r := <-s.reqC: + if r.getReleases != nil { + r.getReleases <- releases + } + case <-t.C: + releases, err = getReleases(ctx) + if err != nil { + log.Printf("Failed to fetch releases: %v", err) + } + } + } +} + +func (s *server) getReleases(ctx context.Context) ([]GLRelease, error) { + sresC := make(chan []GLRelease) + select { + case s.reqC <- &req{getReleases: sresC}: + releases := <-sresC + return releases, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (s *server) handleReleases(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + releases, err := s.getReleases(ctx) + if err != nil { + return + } + + type partition struct { + Name string `json:"name"` + URL string `json:"url"` + Offset string `json:"offset"` + } + + type release struct { + Name string `json:"name"` + Partitions []partition `json:"partitions"` + } + + var resp []release + for _, rel := range releases { + var partitions []partition + offsets := map[string]int64{ + "bootloader": 0, + "partition-table": 0x8000, + "recovery": 0x10000, + "flow3r": 0x90000, + } + for _, pname := range []string{"bootloader", "partition-table", "recovery", "flow3r"} { + partitions = append(partitions, partition{ + Name: pname, + URL: fmt.Sprintf("%srelease/%s/%s.bin", flagBaseURL, rel.TagName, pname), + Offset: fmt.Sprintf("0x%x", offsets[pname]), + }) + } + resp = append(resp, release{ + Name: rel.Name, + Partitions: partitions, + }) + } + w.Header().Add("Content-Type", "application/json") + j := json.NewEncoder(w) + j.Encode(resp) +} + +var ( + reMirrorURL = regexp.MustCompile("^/release/([^/]+)/([^/]+.bin)$") +) + +func (s *server) cacheTarball(rel *GLRelease, data io.Reader) error { + bz2 := bzip2.NewReader(data) + t := tar.NewReader(bz2) + + log.Printf("Extracting %q...", rel.TagName) + cacheEntry := make(map[string][]byte) + for { + hdr, err := t.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("when reading header: %v", err) + } + parts := strings.Split(hdr.Name, "/") + last := parts[len(parts)-1] + log.Printf("%q has %q", rel.TagName, last) + buf := bytes.NewBuffer(nil) + if _, err := io.Copy(buf, t); err != nil { + return fmt.Errorf("file %s: %v", hdr.Name, err) + } + cacheEntry[last] = buf.Bytes() + } + s.cacheMu.Lock() + s.cache[rel.TagName] = cacheEntry + s.cacheMu.Unlock() + return nil +} + +func (s *server) serveMirroredFile(w http.ResponseWriter, r *http.Request, rel *GLRelease, artifact string) { + ctx := r.Context() + + s.cacheMu.RLock() + if s.cache[rel.TagName] != nil && s.cache[rel.TagName][artifact] != nil { + b := s.cache[rel.TagName][artifact] + s.cacheMu.RUnlock() + w.Header().Add("Content-Type", "application/octet-stream") + w.Write(b) + return + } + if s.cache[rel.TagName] != nil { + log.Printf("Failed: no %q in %q", artifact, rel.TagName) + http.NotFound(w, r) + return + } + s.cacheMu.RUnlock() + + log.Printf("Fetching %q (for %q)...", rel.TagName, artifact) + + if len(rel.Assets.Links) != 1 { + log.Printf("Tag %s has %d assets", rel.TagName, len(rel.Assets.Links)) + http.NotFound(w, r) + return + } + link := rel.Assets.Links[0] + req, err := http.NewRequestWithContext(ctx, "GET", link.URL, nil) + if err != nil { + w.WriteHeader(500) + fmt.Fprintf(w, "could not download") + return + } + res, err := http.DefaultTransport.RoundTrip(req) + if err != nil { + w.WriteHeader(500) + fmt.Fprintf(w, "could not download") + return + } + defer res.Body.Close() + + if err := s.cacheTarball(rel, res.Body); err != nil { + w.WriteHeader(500) + fmt.Fprintf(w, "could not cache") + return + } + + s.cacheMu.RLock() + if s.cache[rel.TagName] != nil && s.cache[rel.TagName][artifact] != nil { + b := s.cache[rel.TagName][artifact] + s.cacheMu.RUnlock() + w.Header().Add("Content-Type", "application/octet-stream") + w.Write(b) + return + } + s.cacheMu.RUnlock() + http.NotFound(w, r) +} + +func (s *server) handleReleaseMirror(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + matches := reMirrorURL.FindStringSubmatch(r.URL.Path) + if matches == nil { + http.NotFound(w, r) + return + } + tag := matches[1] + artifact := matches[2] + + releases, err := s.getReleases(ctx) + if err != nil { + return + } + + for _, rel := range releases { + if rel.TagName == tag { + s.serveMirroredFile(w, r, &rel, artifact) + return + } + } +} + +func main() { + flag.StringVar(&flagBaseURL, "base_url", "https://flow3r.garden/api/", "Base address at which this instance runs (used for calculating proxied data URLs)") + flag.StringVar(&flagListen, "listen", ":8080", "Address on which to listen") + flag.StringVar(&flagGitlabHost, "gitlab_host", "git.flow3r.garden", "GitLab instance host") + flag.StringVar(&flagGitlabProject, "gitlab_project", "flow3r/flow3r-firmware", "Name of the organization/project on GitLab") + flag.Parse() + + ctx := context.Background() + s := server{ + reqC: make(chan *req), + cache: make(map[string]map[string][]byte), + } + go s.run(ctx) + + http.HandleFunc("/releases.json", s.handleReleases) + http.HandleFunc("/release/", s.handleReleaseMirror) + log.Printf("Listening on %s...", flagListen) + http.ListenAndServe(flagListen, nil) +}