Why is it so slow?

main.go

@@ -1,23 +1,41 @@
 package main
 
 import (
+	"fmt"
 	log "github.com/sirupsen/logrus"
 	"os"
 )
 
 func init() {
-	log.SetFormatter(&log.JSONFormatter{})
+	// log.SetFormatter(&log.JSONFormatter{})
+	log.SetFormatter(&log.TextFormatter{FullTimestamp: true, DisableColors: false, ForceColors: true})
 	log.SetOutput(os.Stdout)
 	desiredLogLevel := os.Getenv("LOG_LEVEL")
 	level, err := log.ParseLevel(desiredLogLevel)
 	if err != nil {
-		log.Errorf("Failed to parse log level: %s", desiredLogLevel)
+		log.Debugf("Failed to parse log level: %s", desiredLogLevel)
-		log.SetLevel(log.InfoLevel)
+		log.SetLevel(log.WarnLevel)
 	} else {
 		log.SetLevel(level)
 	}
 }
 
 func main() {
-	_, _ = AllStarships()
+	starships, err := AllStarships()
+	if err != nil {
+		log.Errorf("Error loading Starships: %+v", err)
+		os.Exit(1)
+	}
+	data := GetStarshipsPilots(starships)
+	for starship, pilots := range data {
+		if len(pilots) <= 0 {
+			continue
+		}
+		fmt.Println(starship.Name)
+		for _, pilot := range pilots {
+			if pilot != nil {
+				fmt.Println("  " + pilot.Name)
+			}
+		}
+	}
 }

swapi.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	log "github.com/sirupsen/logrus"
 	"net/http"
+	"sync"
 )
 
 const API_PREFIX = "https://swapi.dev/api"
@@ -21,31 +22,22 @@ type Starship struct {
 	// ... other fields unused by this application
 }
 
-// Structs which may be retrieved in a paginated fasion from swapi.dev
+// Represents a single page of data as returned by swapi
-type Pageable interface {
+type PageOf[T interface{}] struct {
-	Starship | Person
+	Count   uint
+	Next    *string
+	Results []T
+	// Previous *string
 }
 
-type Gettable interface {
+// Used for retrieving data from swapi.dev, decoding the unmarshalling the JSON
-	Starship | Person
+// body, and storing the result in the provided struct reference.
-}
-
-// Represents a single page of [Pageable] structs
-type PageOf[T Pageable] struct {
-	Count    uint
-	Next     *string
-	Previous *string
-	Results  []T
-}
-
-// Used for retrieving data from swapi.dev
 //
-//	resp, err := Get("/starships")
+//	myData := interface{}
+//	err := Get("/starships", &myData)
 //
 // If [path] does not start with "/", it assumes you have provided a full URL to
-// make following [PageOf]'s [Next] and [Previous] simpler.
+// make following [PageOf]'s Next and Previous simpler.
-//
-//	resp, err := Get("https://git.lyte.dev/swagger.v1.json")
 func Get[T interface{}](path string, data *T) error {
 	var url string
 	if path[0] == '/' {
@@ -66,31 +58,127 @@ func Get[T interface{}](path string, data *T) error {
 	return err
 }
 
-func AllStarships() (*[]Starship, error) {
+// Retrieves all starships from swapi by fetching the first page and continuing
-	results := make([]Starship, 0)
+// to fetch pages synchronously until no Next page is specified. We collect each
+// request's set of results into a slice.
+func AllStarships() ([]*Starship, error) {
+	results := make([]*Starship, 0)
 	var page PageOf[Starship]
-	var resp *http.Response
 
 	err := Get("/starships", &page)
 	if err != nil {
-		return &[]Starship{}, err
+		return nil, err
 	}
-	err = json.NewDecoder(resp.Body).Decode(&page)
-	log.Debugf("Page: %+v", page)
 	if err != nil {
 		log.Errorf("Error decoding response body: %+v", err)
 	}
-	results = append(results, page.Results...)
+	for r := range page.Results {
-
+		results = append(results, &page.Results[r])
-	// TODO: loop
-	for page.Next != nil {
-		err := Get(*page.Next, &page)
-		log.Debugf("Page: %+v", page)
-		if err != nil {
-			return &[]Starship{}, err
-		}
-		json.NewDecoder(resp.Body).Decode(&page)
-		results = append(results, page.Results...)
 	}
-	return &results, nil
+
+	for page.Next != nil {
+		next := *page.Next
+		page = PageOf[Starship]{}
+		err := Get(next, &page)
+		if err != nil {
+			return nil, err
+		}
+		for r := range page.Results {
+			results = append(results, &page.Results[r])
+		}
+	}
+	return results, nil
+}
+
+// Fetches a [Person].
+func GetPerson(url string) (*Person, error) {
+	person := Person{}
+	err := Get(url, &person)
+	if err != nil {
+		return nil, err
+	} else {
+		return &person, nil
+	}
+}
+
+// Retrieves the set of pilot [Person]s for all [Starship]s simultaneously.
+func GetStarshipsPilots(starships []*Starship) map[*Starship][]*Person {
+	var wg sync.WaitGroup
+
+	log.Infof("Fetching pilots for %d starships...", len(starships))
+
+	personUrlChan := make(chan string)
+	personChan := make(chan *Person)
+
+	fetchedPersons := make(map[string]bool)
+	persons := make(map[string]*Person)
+
+	result := make(map[*Starship][]*Person, len(starships))
+
+	n := 0
+	for _, s := range starships {
+		for _, url := range s.Pilots {
+			if fetchedPersons[url] {
+				continue
+			}
+			fetchedPersons[url] = true
+			wg.Add(1)
+			n += 1
+			go func(url string) {
+				defer wg.Done()
+				person, _ := GetPerson(url)
+				if person != nil {
+					personUrlChan <- url
+					personChan <- person
+				} else {
+					log.Errorf("Failed to fetch person at url %s", url)
+				}
+			}(url)
+		}
+	}
+	go func() {
+		wg.Wait()
+		close(personUrlChan)
+		close(personChan)
+	}()
+	for url := range personUrlChan {
+		persons[url] = <-personChan
+	}
+
+	for _, starship := range starships {
+		pilots := make([]*Person, len(starship.Pilots))
+		for _, url := range starship.Pilots {
+			pilots = append(pilots, persons[url])
+		}
+		result[starship] = pilots
+	}
+
+	return result
+}
+
+// Retrieves all the pilots for a given [Starship] simultaneously
+func GetPilots(starship *Starship) []*Person {
+	var wg sync.WaitGroup
+
+	c := make(chan *Person)
+	for i := range starship.Pilots {
+		wg.Add(1)
+		go func(url string) {
+			person, _ := GetPerson(url)
+			if person != nil {
+				c <- person
+			}
+		}(starship.Pilots[i])
+	}
+
+	go func() {
+		wg.Wait()
+		close(c)
+	}()
+
+	var result []*Person
+	for r := range c {
+		result = append(result, r)
+	}
+	return result
 }