Cache bulk data file

sevenkeys/.gitignore

@@ -1 +1,2 @@
+cache/
 cmd/database/config.json

sevenkeys/cmd/importcards/main.go

@@ -1,13 +1,82 @@
 package main
 
 import (
+	"io"
+	"log"
+	"net/http"
+	"os"
 	"sevenkeys/database"
+	"sevenkeys/database/entities"
 	"sevenkeys/database/operations"
 	"sevenkeys/scryfall/methods"
+	"sevenkeys/scryfall/types"
+	"time"
 )
 
+func check(err error) {
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
 func main() {
+	log.Printf("Getting bulk data listing...")
+	allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards)
+	check(err)
+	log.Printf("Got bulk data listing.")
+
+	log.Printf("Checking if we need to download bulk data...")
+	updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
+	check(err)
+	// Remove the fractional seconds component from the downloaded timestamp because it isn't saved in the cache table
+	updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second)
+
+	db := database.GetDatabaseFromConfig("config.json")
+
+	cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData)
+	check(err)
+
+	if updatedAtTimestamp.After(cachedFileTimestamp) {
+		log.Printf("Cached time: ", cachedFileTimestamp)
+		log.Printf("Update time: ", updatedAtTimestamp)
+		log.Printf("Bulk data has been updated since last cache, redownloading.")
+		cacheDir := "cache"
+
+		err = os.RemoveAll(cacheDir)
+		check(err)
+
+		err := os.Mkdir(cacheDir, os.ModePerm)
+		check(err)
+
+		log.Printf("Downloading bulk card data...")
+		bulkCardsResponse, err := http.Get(allCardsBulkData.DownloadUri)
+		check(err)
+		log.Printf("Downloaded bulk card data.")
+
+		log.Printf("Writing card data to cache file...")
+		cacheFilename := cacheDir + "/all-cards.json"
+		cacheFile, err := os.Create(cacheFilename)
+		check(err)
+
+		defer bulkCardsResponse.Body.Close()
+		defer cacheFile.Close()
+		io.Copy(cacheFile, bulkCardsResponse.Body)
+		log.Printf("Cache file written.")
+
+		log.Printf("Recording timestamp...")
+		err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
+		check(err)
+		log.Printf("Timestamp recorded.")
+	} else {
+		log.Printf("Bulk data has not been updated. Skipping download.")
+	}
+
+	// Marshal the bulk cards json from the cache file into a struct
+	// Import the json into the database
+
+	/* OLD CODE
 	db := database.GetDatabaseFromConfig("config.json")
 	allCards := methods.GetAllCards()
 	operations.InsertCards(db, allCards)
+	*/
 }

sevenkeys/database/entities/cachetimestamp.go

@@ -0,0 +1,3 @@
+package entities
+
+const CacheTypeAllCardsBulkData = "AllCardsBulkData"

sevenkeys/database/operations/inserts.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"log"
 	"sevenkeys/scryfall/types"
+	"time"
 )
 
 var ErrGamepieceExists error = errors.New("Gamepiece already exists in database")
@@ -56,3 +57,20 @@ func InsertCards(db *sql.DB, cards []types.Card) {
 		insertCardPrinting(db, gamepieceId, card.Set, card.ImageUris["png"])
 	}
 }
+
+func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error {
+	query := `INSERT INTO CacheTimestamps (CacheType, Stamp) VALUES (?, ?) ON DUPLICATE KEY UPDATE Stamp = ?;`
+
+	insertOrUpdate, err := db.Prepare(query)
+	defer insertOrUpdate.Close()
+	if err != nil {
+		return err
+	}
+
+	_, err = insertOrUpdate.Exec(cacheType, stamp, stamp)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

sevenkeys/database/operations/selects.go

@@ -3,6 +3,7 @@ package operations
 import (
 	"database/sql"
 	"sevenkeys/database/entities"
+	"time"
 )
 
 func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) {
@@ -13,3 +14,21 @@ func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) {
 
 	return gamepiece, err
 }
+
+func GetCacheTimestampByType(db *sql.DB, cacheType string) (time.Time, error) {
+	var timestamp string
+
+	query := "SELECT Stamp FROM CacheTimestamps WHERE CacheType = ?;"
+	err := db.QueryRow(query, cacheType).Scan(&timestamp)
+
+	if err == sql.ErrNoRows {
+		return time.Unix(0, 0), nil
+	}
+
+	stamp, err := time.Parse("2006-01-02 15:04:05", timestamp)
+	if err != nil {
+		return time.Unix(0, 0), nil
+	}
+
+	return stamp, err
+}

sevenkeys/scryfall/methods/bulkdata.go

@@ -4,76 +4,58 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
-	"log"
 	"net/http"
 	"sevenkeys/scryfall/types"
 )
 
 var BULK_DATA_URI = "https://api.scryfall.com/bulk-data"
 
-func GetBulkData() types.BulkDataList {
+func GetBulkData() (types.BulkDataList, error) {
 	response, err := http.Get(BULK_DATA_URI)
 	if err != nil {
-		log.Fatal(err)
+		return types.BulkDataList{}, err
 	}
 
 	if response.StatusCode != http.StatusOK {
-		log.Fatal(response.StatusCode)
+		return types.BulkDataList{}, errors.New("HTTP request failed with code: " + string(response.StatusCode))
 	}
 
 	defer response.Body.Close()
 	bulkDataBytes, err := io.ReadAll(response.Body)
 	if err != nil {
-		log.Fatal(err)
+		return types.BulkDataList{}, err
 	}
 
 	var bulkDataList types.BulkDataList
 	err = json.Unmarshal(bulkDataBytes, &bulkDataList)
 	if err != nil {
-		log.Fatal(err)
+		return types.BulkDataList{}, err
 	}
 
-	return bulkDataList
+	return bulkDataList, nil
 }
 
-func getBulkDownloadUri(bulkType string) (string, error) {
+func GetBulkDataByType(bulkDataType string) (types.BulkData, error) {
-	bulkDataList := GetBulkData()
+	response, err := http.Get(BULK_DATA_URI + "/" + bulkDataType)
-	for index := range bulkDataList.Data {
-		bulkData := bulkDataList.Data[index]
-		if bulkData.Type == bulkType {
-			return bulkData.DownloadUri, nil
-		}
-	}
-
-	return "", errors.New("No bulk data of type " + bulkType + " found.")
-}
-
-func GetAllCards() []types.Card {
-	allCardsUri, err := getBulkDownloadUri("all_cards")
 	if err != nil {
-		log.Fatal(err)
+		return types.BulkData{}, err
-	}
-
-	response, err := http.Get(allCardsUri)
-	if err != nil {
-		log.Fatal(err)
 	}
 
 	if response.StatusCode != http.StatusOK {
-		log.Fatal(response.StatusCode)
+		return types.BulkData{}, errors.New("HTTP request failed with code: " + string(response.StatusCode))
 	}
 
 	defer response.Body.Close()
-	allCardsBytes, err := io.ReadAll(response.Body)
+	bulkDataBytes, err := io.ReadAll(response.Body)
 	if err != nil {
-		log.Fatal(err)
+		return types.BulkData{}, err
 	}
 
-	var allCards []types.Card
+	var bulkData types.BulkData
-	err = json.Unmarshal(allCardsBytes, &allCards)
+	err = json.Unmarshal(bulkDataBytes, &bulkData)
 	if err != nil {
-		log.Fatal(err)
+		return types.BulkData{}, err
 	}
 
-	return allCards
+	return bulkData, nil
 }

sevenkeys/scryfall/types/bulkdata.go

@@ -1,5 +1,11 @@
 package types
 
+const BulkDataTypeOracleCards string = "oracle_cards"
+const BulkDataTypeUniqueArtwork string = "unique_artwork"
+const BulkDataTypeDefaultCards string = "default_cards"
+const BulkDataTypeAllCards string = "all_cards"
+const BulkDataTypeRulings string = "rulings"
+
 type BulkData struct {
 	Id              string `json:"id"`
 	Uri             string `json:"uri"`

sevenkeys/scryfall/types/timestamp.go

@@ -0,0 +1,3 @@
+package types
+
+const ScryfallTimestampFormat = "2006-01-02T15:04:05.999-07:00"

sevenkeys/sql/createdb.sql

@@ -3,7 +3,7 @@ CREATE DATABASE IF NOT EXISTS sevenkeys;
 USE sevenkeys;
 
 CREATE TABLE IF NOT EXISTS CacheTimestamps (
-	CacheType ENUM('BulkCardPrintings') PRIMARY KEY,
+	CacheType ENUM('AllCardsBulkData') PRIMARY KEY,
 	Stamp DATETIME NOT NULL
 );