Cache bulk data file

This commit is contained in:
The Magician 2024-05-19 15:01:18 +01:00
parent 6def107873
commit c8133803f7
9 changed files with 136 additions and 35 deletions

View File

@ -1 +1,2 @@
cache/
cmd/database/config.json cmd/database/config.json

View File

@ -1,13 +1,82 @@
package main package main
import ( import (
"io"
"log"
"net/http"
"os"
"sevenkeys/database" "sevenkeys/database"
"sevenkeys/database/entities"
"sevenkeys/database/operations" "sevenkeys/database/operations"
"sevenkeys/scryfall/methods" "sevenkeys/scryfall/methods"
"sevenkeys/scryfall/types"
"time"
) )
func check(err error) {
if err != nil {
log.Fatal(err)
}
}
func main() { func main() {
log.Printf("Getting bulk data listing...")
allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards)
check(err)
log.Printf("Got bulk data listing.")
log.Printf("Checking if we need to download bulk data...")
updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
check(err)
// Remove the fractional seconds component from the downloaded timestamp because it isn't saved in the cache table
updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second)
db := database.GetDatabaseFromConfig("config.json")
cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData)
check(err)
if updatedAtTimestamp.After(cachedFileTimestamp) {
log.Printf("Cached time: ", cachedFileTimestamp)
log.Printf("Update time: ", updatedAtTimestamp)
log.Printf("Bulk data has been updated since last cache, redownloading.")
cacheDir := "cache"
err = os.RemoveAll(cacheDir)
check(err)
err := os.Mkdir(cacheDir, os.ModePerm)
check(err)
log.Printf("Downloading bulk card data...")
bulkCardsResponse, err := http.Get(allCardsBulkData.DownloadUri)
check(err)
log.Printf("Downloaded bulk card data.")
log.Printf("Writing card data to cache file...")
cacheFilename := cacheDir + "/all-cards.json"
cacheFile, err := os.Create(cacheFilename)
check(err)
defer bulkCardsResponse.Body.Close()
defer cacheFile.Close()
io.Copy(cacheFile, bulkCardsResponse.Body)
log.Printf("Cache file written.")
log.Printf("Recording timestamp...")
err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
check(err)
log.Printf("Timestamp recorded.")
} else {
log.Printf("Bulk data has not been updated. Skipping download.")
}
// Marshal the bulk cards json from the cache file into a struct
// Import the json into the database
/* OLD CODE
db := database.GetDatabaseFromConfig("config.json") db := database.GetDatabaseFromConfig("config.json")
allCards := methods.GetAllCards() allCards := methods.GetAllCards()
operations.InsertCards(db, allCards) operations.InsertCards(db, allCards)
*/
} }

View File

@ -0,0 +1,3 @@
package entities
const CacheTypeAllCardsBulkData = "AllCardsBulkData"

View File

@ -5,6 +5,7 @@ import (
"errors" "errors"
"log" "log"
"sevenkeys/scryfall/types" "sevenkeys/scryfall/types"
"time"
) )
var ErrGamepieceExists error = errors.New("Gamepiece already exists in database") var ErrGamepieceExists error = errors.New("Gamepiece already exists in database")
@ -56,3 +57,20 @@ func InsertCards(db *sql.DB, cards []types.Card) {
insertCardPrinting(db, gamepieceId, card.Set, card.ImageUris["png"]) insertCardPrinting(db, gamepieceId, card.Set, card.ImageUris["png"])
} }
} }
func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error {
query := `INSERT INTO CacheTimestamps (CacheType, Stamp) VALUES (?, ?) ON DUPLICATE KEY UPDATE Stamp = ?;`
insertOrUpdate, err := db.Prepare(query)
defer insertOrUpdate.Close()
if err != nil {
return err
}
_, err = insertOrUpdate.Exec(cacheType, stamp, stamp)
if err != nil {
return err
}
return nil
}

View File

@ -3,6 +3,7 @@ package operations
import ( import (
"database/sql" "database/sql"
"sevenkeys/database/entities" "sevenkeys/database/entities"
"time"
) )
func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) { func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) {
@ -13,3 +14,21 @@ func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) {
return gamepiece, err return gamepiece, err
} }
func GetCacheTimestampByType(db *sql.DB, cacheType string) (time.Time, error) {
var timestamp string
query := "SELECT Stamp FROM CacheTimestamps WHERE CacheType = ?;"
err := db.QueryRow(query, cacheType).Scan(&timestamp)
if err == sql.ErrNoRows {
return time.Unix(0, 0), nil
}
stamp, err := time.Parse("2006-01-02 15:04:05", timestamp)
if err != nil {
return time.Unix(0, 0), nil
}
return stamp, err
}

View File

@ -4,76 +4,58 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"io" "io"
"log"
"net/http" "net/http"
"sevenkeys/scryfall/types" "sevenkeys/scryfall/types"
) )
var BULK_DATA_URI = "https://api.scryfall.com/bulk-data" var BULK_DATA_URI = "https://api.scryfall.com/bulk-data"
func GetBulkData() types.BulkDataList { func GetBulkData() (types.BulkDataList, error) {
response, err := http.Get(BULK_DATA_URI) response, err := http.Get(BULK_DATA_URI)
if err != nil { if err != nil {
log.Fatal(err) return types.BulkDataList{}, err
} }
if response.StatusCode != http.StatusOK { if response.StatusCode != http.StatusOK {
log.Fatal(response.StatusCode) return types.BulkDataList{}, errors.New("HTTP request failed with code: " + string(response.StatusCode))
} }
defer response.Body.Close() defer response.Body.Close()
bulkDataBytes, err := io.ReadAll(response.Body) bulkDataBytes, err := io.ReadAll(response.Body)
if err != nil { if err != nil {
log.Fatal(err) return types.BulkDataList{}, err
} }
var bulkDataList types.BulkDataList var bulkDataList types.BulkDataList
err = json.Unmarshal(bulkDataBytes, &bulkDataList) err = json.Unmarshal(bulkDataBytes, &bulkDataList)
if err != nil { if err != nil {
log.Fatal(err) return types.BulkDataList{}, err
} }
return bulkDataList return bulkDataList, nil
} }
func getBulkDownloadUri(bulkType string) (string, error) { func GetBulkDataByType(bulkDataType string) (types.BulkData, error) {
bulkDataList := GetBulkData() response, err := http.Get(BULK_DATA_URI + "/" + bulkDataType)
for index := range bulkDataList.Data {
bulkData := bulkDataList.Data[index]
if bulkData.Type == bulkType {
return bulkData.DownloadUri, nil
}
}
return "", errors.New("No bulk data of type " + bulkType + " found.")
}
func GetAllCards() []types.Card {
allCardsUri, err := getBulkDownloadUri("all_cards")
if err != nil { if err != nil {
log.Fatal(err) return types.BulkData{}, err
}
response, err := http.Get(allCardsUri)
if err != nil {
log.Fatal(err)
} }
if response.StatusCode != http.StatusOK { if response.StatusCode != http.StatusOK {
log.Fatal(response.StatusCode) return types.BulkData{}, errors.New("HTTP request failed with code: " + string(response.StatusCode))
} }
defer response.Body.Close() defer response.Body.Close()
allCardsBytes, err := io.ReadAll(response.Body) bulkDataBytes, err := io.ReadAll(response.Body)
if err != nil { if err != nil {
log.Fatal(err) return types.BulkData{}, err
} }
var allCards []types.Card var bulkData types.BulkData
err = json.Unmarshal(allCardsBytes, &allCards) err = json.Unmarshal(bulkDataBytes, &bulkData)
if err != nil { if err != nil {
log.Fatal(err) return types.BulkData{}, err
} }
return allCards return bulkData, nil
} }

View File

@ -1,5 +1,11 @@
package types package types
const BulkDataTypeOracleCards string = "oracle_cards"
const BulkDataTypeUniqueArtwork string = "unique_artwork"
const BulkDataTypeDefaultCards string = "default_cards"
const BulkDataTypeAllCards string = "all_cards"
const BulkDataTypeRulings string = "rulings"
type BulkData struct { type BulkData struct {
Id string `json:"id"` Id string `json:"id"`
Uri string `json:"uri"` Uri string `json:"uri"`

View File

@ -0,0 +1,3 @@
package types
const ScryfallTimestampFormat = "2006-01-02T15:04:05.999-07:00"

View File

@ -3,7 +3,7 @@ CREATE DATABASE IF NOT EXISTS sevenkeys;
USE sevenkeys; USE sevenkeys;
CREATE TABLE IF NOT EXISTS CacheTimestamps ( CREATE TABLE IF NOT EXISTS CacheTimestamps (
CacheType ENUM('BulkCardPrintings') PRIMARY KEY, CacheType ENUM('AllCardsBulkData') PRIMARY KEY,
Stamp DATETIME NOT NULL Stamp DATETIME NOT NULL
); );