From c8133803f759706957a2310f7f57bf4a54a3c35e Mon Sep 17 00:00:00 2001 From: The Magician Date: Sun, 19 May 2024 15:01:18 +0100 Subject: [PATCH] Cache bulk data file --- sevenkeys/.gitignore | 1 + sevenkeys/cmd/importcards/main.go | 69 +++++++++++++++++++ sevenkeys/database/entities/cachetimestamp.go | 3 + sevenkeys/database/operations/inserts.go | 18 +++++ sevenkeys/database/operations/selects.go | 19 +++++ sevenkeys/scryfall/methods/bulkdata.go | 50 +++++--------- sevenkeys/scryfall/types/bulkdata.go | 6 ++ sevenkeys/scryfall/types/timestamp.go | 3 + sevenkeys/sql/createdb.sql | 2 +- 9 files changed, 136 insertions(+), 35 deletions(-) create mode 100644 sevenkeys/database/entities/cachetimestamp.go create mode 100644 sevenkeys/scryfall/types/timestamp.go diff --git a/sevenkeys/.gitignore b/sevenkeys/.gitignore index 4fb88ac..c70c1c1 100644 --- a/sevenkeys/.gitignore +++ b/sevenkeys/.gitignore @@ -1 +1,2 @@ +cache/ cmd/database/config.json diff --git a/sevenkeys/cmd/importcards/main.go b/sevenkeys/cmd/importcards/main.go index 842722a..80f8702 100644 --- a/sevenkeys/cmd/importcards/main.go +++ b/sevenkeys/cmd/importcards/main.go @@ -1,13 +1,82 @@ package main import ( + "io" + "log" + "net/http" + "os" "sevenkeys/database" + "sevenkeys/database/entities" "sevenkeys/database/operations" "sevenkeys/scryfall/methods" + "sevenkeys/scryfall/types" + "time" ) +func check(err error) { + if err != nil { + log.Fatal(err) + } +} + func main() { + log.Printf("Getting bulk data listing...") + allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards) + check(err) + log.Printf("Got bulk data listing.") + + log.Printf("Checking if we need to download bulk data...") + updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt) + check(err) + // Remove the fractional seconds component from the downloaded timestamp because it isn't saved in the cache table + updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second) + + db := database.GetDatabaseFromConfig("config.json") + + cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData) + check(err) + + if updatedAtTimestamp.After(cachedFileTimestamp) { + log.Printf("Cached time: ", cachedFileTimestamp) + log.Printf("Update time: ", updatedAtTimestamp) + log.Printf("Bulk data has been updated since last cache, redownloading.") + cacheDir := "cache" + + err = os.RemoveAll(cacheDir) + check(err) + + err := os.Mkdir(cacheDir, os.ModePerm) + check(err) + + log.Printf("Downloading bulk card data...") + bulkCardsResponse, err := http.Get(allCardsBulkData.DownloadUri) + check(err) + log.Printf("Downloaded bulk card data.") + + log.Printf("Writing card data to cache file...") + cacheFilename := cacheDir + "/all-cards.json" + cacheFile, err := os.Create(cacheFilename) + check(err) + + defer bulkCardsResponse.Body.Close() + defer cacheFile.Close() + io.Copy(cacheFile, bulkCardsResponse.Body) + log.Printf("Cache file written.") + + log.Printf("Recording timestamp...") + err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp) + check(err) + log.Printf("Timestamp recorded.") + } else { + log.Printf("Bulk data has not been updated. Skipping download.") + } + + // Marshal the bulk cards json from the cache file into a struct + // Import the json into the database + + /* OLD CODE db := database.GetDatabaseFromConfig("config.json") allCards := methods.GetAllCards() operations.InsertCards(db, allCards) + */ } diff --git a/sevenkeys/database/entities/cachetimestamp.go b/sevenkeys/database/entities/cachetimestamp.go new file mode 100644 index 0000000..7871b00 --- /dev/null +++ b/sevenkeys/database/entities/cachetimestamp.go @@ -0,0 +1,3 @@ +package entities + +const CacheTypeAllCardsBulkData = "AllCardsBulkData" diff --git a/sevenkeys/database/operations/inserts.go b/sevenkeys/database/operations/inserts.go index b718aa8..8302eb4 100644 --- a/sevenkeys/database/operations/inserts.go +++ b/sevenkeys/database/operations/inserts.go @@ -5,6 +5,7 @@ import ( "errors" "log" "sevenkeys/scryfall/types" + "time" ) var ErrGamepieceExists error = errors.New("Gamepiece already exists in database") @@ -56,3 +57,20 @@ func InsertCards(db *sql.DB, cards []types.Card) { insertCardPrinting(db, gamepieceId, card.Set, card.ImageUris["png"]) } } + +func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error { + query := `INSERT INTO CacheTimestamps (CacheType, Stamp) VALUES (?, ?) ON DUPLICATE KEY UPDATE Stamp = ?;` + + insertOrUpdate, err := db.Prepare(query) + defer insertOrUpdate.Close() + if err != nil { + return err + } + + _, err = insertOrUpdate.Exec(cacheType, stamp, stamp) + if err != nil { + return err + } + + return nil +} diff --git a/sevenkeys/database/operations/selects.go b/sevenkeys/database/operations/selects.go index 05becf6..651256a 100644 --- a/sevenkeys/database/operations/selects.go +++ b/sevenkeys/database/operations/selects.go @@ -3,6 +3,7 @@ package operations import ( "database/sql" "sevenkeys/database/entities" + "time" ) func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) { @@ -13,3 +14,21 @@ func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) { return gamepiece, err } + +func GetCacheTimestampByType(db *sql.DB, cacheType string) (time.Time, error) { + var timestamp string + + query := "SELECT Stamp FROM CacheTimestamps WHERE CacheType = ?;" + err := db.QueryRow(query, cacheType).Scan(×tamp) + + if err == sql.ErrNoRows { + return time.Unix(0, 0), nil + } + + stamp, err := time.Parse("2006-01-02 15:04:05", timestamp) + if err != nil { + return time.Unix(0, 0), nil + } + + return stamp, err +} diff --git a/sevenkeys/scryfall/methods/bulkdata.go b/sevenkeys/scryfall/methods/bulkdata.go index 73a48e1..2856b62 100644 --- a/sevenkeys/scryfall/methods/bulkdata.go +++ b/sevenkeys/scryfall/methods/bulkdata.go @@ -4,76 +4,58 @@ import ( "encoding/json" "errors" "io" - "log" "net/http" "sevenkeys/scryfall/types" ) var BULK_DATA_URI = "https://api.scryfall.com/bulk-data" -func GetBulkData() types.BulkDataList { +func GetBulkData() (types.BulkDataList, error) { response, err := http.Get(BULK_DATA_URI) if err != nil { - log.Fatal(err) + return types.BulkDataList{}, err } if response.StatusCode != http.StatusOK { - log.Fatal(response.StatusCode) + return types.BulkDataList{}, errors.New("HTTP request failed with code: " + string(response.StatusCode)) } defer response.Body.Close() bulkDataBytes, err := io.ReadAll(response.Body) if err != nil { - log.Fatal(err) + return types.BulkDataList{}, err } var bulkDataList types.BulkDataList err = json.Unmarshal(bulkDataBytes, &bulkDataList) if err != nil { - log.Fatal(err) + return types.BulkDataList{}, err } - return bulkDataList + return bulkDataList, nil } -func getBulkDownloadUri(bulkType string) (string, error) { - bulkDataList := GetBulkData() - for index := range bulkDataList.Data { - bulkData := bulkDataList.Data[index] - if bulkData.Type == bulkType { - return bulkData.DownloadUri, nil - } - } - - return "", errors.New("No bulk data of type " + bulkType + " found.") -} - -func GetAllCards() []types.Card { - allCardsUri, err := getBulkDownloadUri("all_cards") +func GetBulkDataByType(bulkDataType string) (types.BulkData, error) { + response, err := http.Get(BULK_DATA_URI + "/" + bulkDataType) if err != nil { - log.Fatal(err) - } - - response, err := http.Get(allCardsUri) - if err != nil { - log.Fatal(err) + return types.BulkData{}, err } if response.StatusCode != http.StatusOK { - log.Fatal(response.StatusCode) + return types.BulkData{}, errors.New("HTTP request failed with code: " + string(response.StatusCode)) } defer response.Body.Close() - allCardsBytes, err := io.ReadAll(response.Body) + bulkDataBytes, err := io.ReadAll(response.Body) if err != nil { - log.Fatal(err) + return types.BulkData{}, err } - var allCards []types.Card - err = json.Unmarshal(allCardsBytes, &allCards) + var bulkData types.BulkData + err = json.Unmarshal(bulkDataBytes, &bulkData) if err != nil { - log.Fatal(err) + return types.BulkData{}, err } - return allCards + return bulkData, nil } diff --git a/sevenkeys/scryfall/types/bulkdata.go b/sevenkeys/scryfall/types/bulkdata.go index a9976e8..29c4e3b 100644 --- a/sevenkeys/scryfall/types/bulkdata.go +++ b/sevenkeys/scryfall/types/bulkdata.go @@ -1,5 +1,11 @@ package types +const BulkDataTypeOracleCards string = "oracle_cards" +const BulkDataTypeUniqueArtwork string = "unique_artwork" +const BulkDataTypeDefaultCards string = "default_cards" +const BulkDataTypeAllCards string = "all_cards" +const BulkDataTypeRulings string = "rulings" + type BulkData struct { Id string `json:"id"` Uri string `json:"uri"` diff --git a/sevenkeys/scryfall/types/timestamp.go b/sevenkeys/scryfall/types/timestamp.go new file mode 100644 index 0000000..0c93965 --- /dev/null +++ b/sevenkeys/scryfall/types/timestamp.go @@ -0,0 +1,3 @@ +package types + +const ScryfallTimestampFormat = "2006-01-02T15:04:05.999-07:00" diff --git a/sevenkeys/sql/createdb.sql b/sevenkeys/sql/createdb.sql index 9d3a518..4e02eab 100644 --- a/sevenkeys/sql/createdb.sql +++ b/sevenkeys/sql/createdb.sql @@ -3,7 +3,7 @@ CREATE DATABASE IF NOT EXISTS sevenkeys; USE sevenkeys; CREATE TABLE IF NOT EXISTS CacheTimestamps ( - CacheType ENUM('BulkCardPrintings') PRIMARY KEY, + CacheType ENUM('AllCardsBulkData') PRIMARY KEY, Stamp DATETIME NOT NULL );