Cache bulk data file

sevenkeys/cmd/importcards/main.go

@@ -2,17 +2,10 @@ package main
 
 import (
 	"encoding/json"
-	"io"
 	"io/ioutil"
 	"log"
-	"net/http"
-	"os"
-	"sevenkeys/database"
-	"sevenkeys/database/entities"
 	"sevenkeys/database/operations"
-	"sevenkeys/scryfall/methods"
 	"sevenkeys/scryfall/types"
-	"time"
 )
 
 func check(err error) {
@@ -22,49 +15,7 @@ func check(err error) {
 }
 
 func main() {
-	log.Printf("Getting bulk data listing...")
-	allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards)
-	check(err)
-	log.Printf("Got bulk data listing.")
-
-	log.Printf("Checking if we need to download bulk data...")
-	updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
-	check(err)
-	// Remove the fractional seconds component from the downloaded timestamp because it isn't saved in the cache table
-	updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second)
-
-	db := database.GetDatabaseFromConfig("config.json")
-
-	cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData)
-	check(err)
-
 	if updatedAtTimestamp.After(cachedFileTimestamp) {
-		log.Printf("Bulk data has been updated since last cache, redownloading.")
-
-		err = os.RemoveAll(CACHE_DIR)
-		check(err)
-
-		err := os.Mkdir(CACHE_DIR, os.ModePerm)
-		check(err)
-
-		log.Printf("Downloading bulk card data...")
-		bulkCardsResponse, err := http.Get(allCardsBulkData.DownloadUri)
-		check(err)
-		log.Printf("Downloaded bulk card data.")
-
-		log.Printf("Writing card data to cache file...")
-		cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
-		check(err)
-
-		defer bulkCardsResponse.Body.Close()
-		defer cacheFile.Close()
-		io.Copy(cacheFile, bulkCardsResponse.Body)
-		log.Printf("Cache file written.")
-
-		log.Printf("Recording timestamp...")
-		err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
-		check(err)
-		log.Printf("Timestamp recorded.")
 	} else {
 		log.Printf("Bulk data has not been updated. Skipping download.")
 	}

sevenkeys/main.go

@@ -7,9 +7,11 @@ import (
 	"net/http"
 	"os"
 	"sevenkeys/database"
+	"sevenkeys/database/entities"
 	"sevenkeys/database/operations"
 	"sevenkeys/scryfall/methods"
 	"sevenkeys/scryfall/types"
+	"time"
 )
 
 const CACHE_DIR string = "cache"
@@ -73,6 +75,44 @@ func importSets(db *sql.DB, sets []types.Set) error {
 	return nil
 }
 
+func cardsUpdatedSinceCache(db *sql.DB, updatedAtTimestamp time.Time) (bool, error) {
+	cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData)
+	if err != nil {
+		return false, err
+	}
+
+	return updatedAtTimestamp.After(cachedFileTimestamp), nil
+}
+
+func cacheAllCardsFile(db *sql.DB, uri string, updatedAtTimestamp time.Time) error {
+	log.Printf("Downloading bulk card data...")
+	bulkCardsResponse, err := http.Get(uri)
+	if err != nil {
+		return err
+	}
+	log.Printf("Downloaded bulk card data.")
+
+	log.Printf("Writing card data to cache file...")
+	cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
+	if err != nil {
+		return err
+	}
+
+	defer bulkCardsResponse.Body.Close()
+	defer cacheFile.Close()
+	io.Copy(cacheFile, bulkCardsResponse.Body)
+	log.Printf("Cache file written.")
+
+	log.Printf("Recording timestamp...")
+	err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
+	if err != nil {
+		return err
+	}
+	log.Printf("Timestamp recorded.")
+
+	return nil
+}
+
 func main() {
 	log.Println("Connecting to database...")
 	db := database.GetDatabaseFromConfig("config.json")
@@ -83,18 +123,39 @@ func main() {
 	check(err)
 	log.Println("Created cache directories.")
 
-	log.Println("Downloading set data from Scryfall...")
+	/*
-	sets, err := methods.GetSets()
+		log.Println("Downloading set data from Scryfall...")
-	check(err)
+		sets, err := methods.GetSets()
-	log.Println("Downloaded set data.")
+		check(err)
+		log.Println("Downloaded set data.")
 
-	log.Println("Importing set data...")
+		log.Println("Importing set data...")
-	err = importSets(db, sets)
+		err = importSets(db, sets)
-	check(err)
+		check(err)
-	log.Println("Imported sets.")
+		log.Println("Imported sets.")
+	*/
 
-	// Check whether card data has been updated since last download
+	log.Println("Downloading bulk data entry...")
-	// If yes, redownload and recache the bulk data
+	allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards)
+	check(err)
+	log.Println("Downloaded bulk data entry.")
+
+	updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
+	check(err)
+	updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second) // Remove the fractional seconds component from the timestamp
+
+	log.Println("Checking for redownload...")
+	updated, err := cardsUpdatedSinceCache(db, updatedAtTimestamp)
+	check(err)
+
+	if updated {
+		log.Printf("Bulk data has been updated since last cache, redownloading.")
+		return
+		err = cacheAllCardsFile(db, allCardsBulkData.DownloadUri, updatedAtTimestamp)
+		check(err)
+	} else {
+		log.Printf("Bulk data has not been updated. Skipping download.")
+	}
 
 	// Unmarshal cached file into Golang struct
 	// Import cards into database