Refactor import/update code

This commit is contained in:
The Magician 2024-05-29 20:55:38 +01:00
parent 708bfdf90b
commit f84f19a0b7
13 changed files with 313 additions and 285 deletions

View File

@ -1,6 +1,9 @@
package database package database
import "database/sql" import (
"database/sql"
"time"
)
const CacheTypeAllCardsBulkData = "AllCardsBulkData" const CacheTypeAllCardsBulkData = "AllCardsBulkData"
@ -12,3 +15,23 @@ func GetCacheTimestampByType(db *sql.DB, cacheType string) (string, error) {
return timestamp, err return timestamp, err
} }
func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error {
query := `INSERT INTO CacheTimestamp (CacheType, Stamp)
VALUES (?, ?)
ON DUPLICATE KEY
UPDATE Stamp = ?;`
insertOrUpdate, err := db.Prepare(query)
defer insertOrUpdate.Close()
if err != nil {
return err
}
_, err = insertOrUpdate.Exec(cacheType, stamp, stamp)
if err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,39 @@
package database
import (
"database/sql"
)
type CardPrinting struct {
Id int
Name string
SetCode string
IsFoil bool
IsPromo bool
CollectorNumber string
Language string
}
func InsertCardPrinting(db *sql.DB, cardPrinting CardPrinting) error {
query := `INSERT INTO CardPrinting (
Name,
SetCode,
IsFoil,
IsPromo,
CollectorNumber,
Language)
VALUES (?, ?, ?, ?, ?, ?);`
insert, err := db.Prepare(query)
defer insert.Close()
if err != nil {
return err
}
_, err = insert.Exec(cardPrinting.Name, cardPrinting.SetCode, cardPrinting.IsFoil, cardPrinting.IsPromo, cardPrinting.CollectorNumber, cardPrinting.Language)
if err != nil {
return err
}
return nil
}

View File

@ -2,70 +2,8 @@ package operations
import ( import (
"database/sql" "database/sql"
"sevenkeys/logic/scryfall/types"
"time"
) )
func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error {
query := `INSERT INTO CacheTimestamp (CacheType, Stamp)
VALUES (?, ?)
ON DUPLICATE KEY
UPDATE Stamp = ?;`
insertOrUpdate, err := db.Prepare(query)
defer insertOrUpdate.Close()
if err != nil {
return err
}
_, err = insertOrUpdate.Exec(cacheType, stamp, stamp)
if err != nil {
return err
}
return nil
}
// TODO: There's really no need for this to be an "upsert", we can just ignore sets that we already have
func InsertOrUpdateSet(db *sql.DB, set types.Set) error {
query := `INSERT INTO ExpansionSet (SetCode, Name, CardCount, IconSvgUri)
VALUES (?, ?, ?, ?)
ON DUPLICATE KEY
Update Name = ?, CardCount = ?, IconSvgUri = ?;`
insertOrUpdate, err := db.Prepare(query)
defer insertOrUpdate.Close()
if err != nil {
return err
}
_, err = insertOrUpdate.Exec(set.Code, set.Name, set.CardCount, set.IconSvgUri, set.Name, set.CardCount, set.IconSvgUri)
if err != nil {
return err
}
return nil
}
func InsertCard(db *sql.DB, card types.Card) error {
query := `INSERT IGNORE INTO CardPrinting
(Id, Name, SetCode, HasFoil, HasNonFoil, IsReserved, IsRacist, IsPromo, CollectorNumber, Language)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`
insert, err := db.Prepare(query)
defer insert.Close()
if err != nil {
return err
}
_, err = insert.Exec(card.Id, card.Name, card.Set, card.Foil, card.NonFoil, card.Reserved, card.ContentWarning, card.Promo, card.CollectorNumber, card.Language)
if err != nil {
return err
}
return nil
}
func InsertCardStorageLocation(db *sql.DB, cardPrintingId string, isFoil bool, storageBox string, source string) error { func InsertCardStorageLocation(db *sql.DB, cardPrintingId string, isFoil bool, storageBox string, source string) error {
var lastPosition int var lastPosition int
getLastPositionQuery := `SELECT Position FROM CardStorageLocation WHERE StorageBox = ? ORDER BY Position DESC LIMIT 1;` getLastPositionQuery := `SELECT Position FROM CardStorageLocation WHERE StorageBox = ? ORDER BY Position DESC LIMIT 1;`

23
sevenkeys/database/set.go Normal file
View File

@ -0,0 +1,23 @@
package database
import (
"database/sql"
"sevenkeys/logic/scryfall"
)
func InsertSet(db *sql.DB, set scryfall.Set) error {
query := `INSERT INTO ExpansionSet (SetCode, Name, CardCount, IconSvgUri) VALUES (?, ?, ?, ?);`
insert, err := db.Prepare(query)
defer insert.Close()
if err != nil {
return err
}
_, err = insert.Exec(set.Code, set.Name, set.CardCount, set.IconSvgUri)
if err != nil {
return err
}
return nil
}

View File

@ -15,25 +15,22 @@ CREATE TABLE IF NOT EXISTS ExpansionSet (
); );
CREATE TABLE IF NOT EXISTS CardPrinting ( CREATE TABLE IF NOT EXISTS CardPrinting (
Id VARCHAR(36) PRIMARY KEY, -- GUID Id INT AUTO_INCREMENT PRIMARY KEY,
Name VARCHAR(150) NOT NULL, Name VARCHAR(150) NOT NULL,
SetCode VARCHAR(6) NOT NULL, SetCode VARCHAR(6) NOT NULL,
FOREIGN KEY (SetCode) REFERENCES ExpansionSet(SetCode), FOREIGN KEY (SetCode) REFERENCES ExpansionSet(SetCode),
HasFoil BOOLEAN NOT NULL, IsFoil BOOLEAN NOT NULL,
HasNonFoil BOOLEAN NOT NULL,
IsReserved BOOLEAN NOT NULL,
IsRacist BOOLEAN NOT NULL,
IsPromo BOOLEAN NOT NULL, IsPromo BOOLEAN NOT NULL,
CollectorNumber VARCHAR(10) NOT NULL, CollectorNumber VARCHAR(10) NOT NULL,
Language VARCHAR(3) NOT NULL Language VARCHAR(3) NOT NULL
); );
CREATE TABLE IF NOT EXISTS CardStorageLocation ( -- CREATE TABLE IF NOT EXISTS CardStorageLocation (
Id INT AUTO_INCREMENT PRIMARY KEY, -- Id INT AUTO_INCREMENT PRIMARY KEY,
CardPrintingId VARCHAR(36) NOT NULL, -- CardPrintingId VARCHAR(36) NOT NULL,
FOREIGN KEY (CardPrintingId) REFERENCES CardPrinting(Id), -- FOREIGN KEY (CardPrintingId) REFERENCES CardPrinting(Id),
IsFoil BOOLEAN NOT NULL, -- IsFoil BOOLEAN NOT NULL,
StorageBox VARCHAR(20) NOT NULL, -- StorageBox VARCHAR(20) NOT NULL,
Source VARCHAR(100) NULL, -- Source VARCHAR(100) NULL,
Position INT NOT NULL -- Position INT NOT NULL
); -- );

View File

@ -6,6 +6,7 @@ require github.com/go-sql-driver/mysql v1.8.1
require ( require (
filippo.io/edwards25519 v1.1.0 // indirect filippo.io/edwards25519 v1.1.0 // indirect
github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0 // indirect
github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-cmp v0.5.9 // indirect
gotest.tools/v3 v3.5.1 // indirect gotest.tools/v3 v3.5.1 // indirect
) )

View File

@ -1,5 +1,7 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0 h1:meiLwrW6ukHHehydhoDxVHdQKQe7TFgEpH0A0hHBAWs=
github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0/go.mod h1:ZH8V0509n2OSZLMYTMHzcy4hqUB+rG8ghK1zsP4i5gE=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=

View File

@ -1,175 +0,0 @@
package main
import (
"database/sql"
"encoding/json"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"sevenkeys/database/entities"
"sevenkeys/database/operations"
"sevenkeys/logic/scryfall/types"
"time"
)
const CACHE_DIR string = "cache"
const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/"
const SET_ICON_FILE_EXTENSION string = ".svg"
const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json"
const GAME_PAPER = "paper"
func check(err error) {
if err != nil {
log.Fatal(err)
}
}
func createCacheDirectories() error {
err := os.Mkdir(CACHE_DIR, os.ModePerm)
if err != nil && !os.IsExist(err) {
return err
}
err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm)
if err != nil && !os.IsExist(err) {
return err
}
return nil
}
func importSets(db *sql.DB, sets []types.Set) error {
for _, set := range sets {
// We're only interested in paper cards, so skip importing
// any sets that were only released in a video game
if set.Digital {
continue
}
log.Println("Importing " + set.Code)
err := operations.InsertOrUpdateSet(db, set)
if err != nil {
return err
}
log.Println("Downloading logo for " + set.Code)
response, err := http.Get(set.IconSvgUri)
defer response.Body.Close()
if err != nil {
return err
}
iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION
iconFile, err := os.Create(iconFilename)
if err != nil {
return err
}
io.Copy(iconFile, response.Body)
log.Println("Finished importing " + set.Code)
}
return nil
}
func cacheAllCardsFile(db *sql.DB, uri string, updatedAtTimestamp time.Time) error {
log.Printf("Downloading bulk card data...")
bulkCardsResponse, err := http.Get(uri)
if err != nil {
return err
}
log.Printf("Downloaded bulk card data.")
log.Printf("Writing card data to cache file...")
cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
if err != nil {
return err
}
defer bulkCardsResponse.Body.Close()
defer cacheFile.Close()
io.Copy(cacheFile, bulkCardsResponse.Body)
log.Printf("Cache file written.")
log.Printf("Recording timestamp...")
err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
if err != nil {
return err
}
log.Printf("Timestamp recorded.")
return nil
}
func importCards(db *sql.DB, cards []types.Card) error {
for _, card := range cards {
// We're only interested in paper cards, so skip cards or printings of a card which
// aren't available in paper
if !isPaper(card) {
continue
}
err := operations.InsertCard(db, card)
if err != nil {
return err
}
}
return nil
}
func isPaper(card types.Card) bool {
var paper bool = false
for _, game := range card.Games {
if game == GAME_PAPER {
paper = true
}
}
return paper
}
func main() {
log.Println("Creating cache directories...")
err := createCacheDirectories()
check(err)
log.Println("Created cache directories.")
log.Println("Downloading set data from Scryfall...")
sets, err := methods.GetSets()
check(err)
log.Println("Downloaded set data.")
log.Println("Importing set data...")
err = importSets(db, sets)
check(err)
log.Println("Imported sets.")
if updated {
log.Printf("Bulk data has been updated since last cache, redownloading.")
err = cacheAllCardsFile(db, allCardsBulkData.DownloadUri, updatedAtTimestamp)
check(err)
} else {
log.Printf("Bulk data has not been updated. Skipping download.")
}
log.Printf("Unmarsaling file into slice...")
allCardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME)
check(err)
var allCards []types.Card
err = json.Unmarshal(allCardsBytes, &allCards)
check(err)
log.Printf("Unmarshaled file.")
log.Printf("Importing card data into database...")
err = importCards(db, allCards)
check(err)
log.Printf("Imported card data.")
}

View File

@ -5,19 +5,21 @@ import (
"errors" "errors"
"io" "io"
"net/http" "net/http"
"time"
) )
type BulkData struct { type BulkData struct {
Id string `json:"id"` Id string `json:"id"`
Uri string `json:"uri"` Uri string `json:"uri"`
Type string `json:"type"` Type string `json:"type"`
Name string `json:"name"` Name string `json:"name"`
Description string `json:"description"` Description string `json:"description"`
DownloadUri string `json:"download_uri"` DownloadUri string `json:"download_uri"`
UpdatedAt string `json:"updated_at"` UpdatedAt string `json:"updated_at"`
Size int `json:"size"` UpdatedAtTime time.Time `json:"ignore"`
ContentType string `json:"content_type"` Size int `json:"size"`
ContentEncoding string `json:"content_encoding"` ContentType string `json:"content_type"`
ContentEncoding string `json:"content_encoding"`
} }
const BULK_DATA_URI = "https://api.scryfall.com/bulk-data" const BULK_DATA_URI = "https://api.scryfall.com/bulk-data"
@ -50,5 +52,12 @@ func GetBulkDataByType(bulkDataType string) (BulkData, error) {
return BulkData{}, err return BulkData{}, err
} }
bulkData.UpdatedAtTime, err = time.Parse(ScryfallTimestampFormat, bulkData.UpdatedAt)
if err != nil {
return BulkData{}, err
}
// Round to the nearest second; this is so that comparison with the timestamp stored in the database works as intended
bulkData.UpdatedAtTime = bulkData.UpdatedAtTime.Truncate(time.Second)
return bulkData, nil return bulkData, nil
} }

View File

@ -1,14 +1,11 @@
package scryfall package scryfall
type Card struct { type Card struct {
Id string `json:"id"` // GUID
Name string `json:"name"` Name string `json:"name"`
Set string `json:"set"` Set string `json:"set"`
Games []string `json:"games"` Games []string `json:"games"`
Foil bool `json:"foil"` Foil bool `json:"foil"`
NonFoil bool `json:"nonfoil"` NonFoil bool `json:"nonfoil"`
Reserved bool `json:"reserved"`
ContentWarning bool `json:"content_warning,omitempty"`
Promo bool `json:"promo"` Promo bool `json:"promo"`
CollectorNumber string `json:"collector_number"` CollectorNumber string `json:"collector_number"`
Language string `json:"lang"` Language string `json:"lang"`

View File

@ -23,7 +23,7 @@ type Set struct {
const SETS_API_URL string = "https://api.scryfall.com/sets" const SETS_API_URL string = "https://api.scryfall.com/sets"
func GetSets() ([]Set, error) { func GetAllSets() ([]Set, error) {
response, err := http.Get(SETS_API_URL) response, err := http.Get(SETS_API_URL)
if err != nil { if err != nil {
return []Set{}, nil return []Set{}, nil

View File

@ -2,14 +2,27 @@ package logic
import ( import (
"database/sql" "database/sql"
"encoding/json"
"fmt" "fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"sevenkeys/database" "sevenkeys/database"
"sevenkeys/logic/scryfall" "sevenkeys/logic/scryfall"
"strings" "strings"
"time" "time"
sqlerr "github.com/go-mysql/errors"
) )
func CheckForUpdates(db *sql.DB) (bool, error) { const GAME_PAPER = "paper"
func CheckForUpdates(db *sql.DB, bulkData scryfall.BulkData) (bool, error) {
// TODO: We also want to update if:
// - there is no cached version of the all-cards.json file or it is empty
// - The set icon files are missing
cachedFileTimestampStr, err := database.GetCacheTimestampByType(db, database.CacheTypeAllCardsBulkData) cachedFileTimestampStr, err := database.GetCacheTimestampByType(db, database.CacheTypeAllCardsBulkData)
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return true, nil return true, nil
@ -19,18 +32,7 @@ func CheckForUpdates(db *sql.DB) (bool, error) {
cachedFileTimestamp, err := time.Parse("2006-01-02 15:04:05", cachedFileTimestampStr) cachedFileTimestamp, err := time.Parse("2006-01-02 15:04:05", cachedFileTimestampStr)
allCardsBulkData, err := scryfall.GetBulkDataByType(scryfall.BulkDataTypeAllCards) return bulkData.UpdatedAtTime.After(cachedFileTimestamp), nil
if err != nil {
return false, err
}
bulkCardsUpdatedTimestamp, err := time.Parse(scryfall.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
if err != nil {
return false, err
}
bulkCardsUpdatedTimestamp = bulkCardsUpdatedTimestamp.Truncate(time.Second)
return bulkCardsUpdatedTimestamp.After(cachedFileTimestamp), nil
} }
func ConfirmUpdate() bool { func ConfirmUpdate() bool {
@ -40,3 +42,162 @@ func ConfirmUpdate() bool {
return strings.ToUpper(response) == "Y" return strings.ToUpper(response) == "Y"
} }
func UpdateSets(db *sql.DB) error {
sets, err := scryfall.GetAllSets()
if err != nil {
return err
}
for _, set := range sets {
// We're only interested in paper cards, so skip importing
// any sets that were only released in a video game
if set.Digital {
continue
}
err := database.InsertSet(db, set)
// If we already have this set in the database, then we can just skip it
if ok, insertErr := sqlerr.Error(err); ok {
if insertErr == sqlerr.ErrDupeKey {
continue
}
}
if err != nil {
return err
}
response, err := http.Get(set.IconSvgUri)
defer response.Body.Close()
if err != nil {
return nil
}
iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION
iconFile, err := os.Create(iconFilename)
if err != nil {
return err
}
io.Copy(iconFile, response.Body)
}
fmt.Println("Sets updated.")
return nil
}
func cacheBulkCardsFile(db *sql.DB, bulkData scryfall.BulkData) error {
bulkCardsResponse, err := http.Get(bulkData.DownloadUri)
defer bulkCardsResponse.Body.Close()
if err != nil {
return err
}
cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
defer cacheFile.Close()
if err != nil {
return err
}
io.Copy(cacheFile, bulkCardsResponse.Body)
err = database.InsertOrUpdateCacheTimestampByType(db, database.CacheTypeAllCardsBulkData, bulkData.UpdatedAtTime)
if err != nil {
return err
}
return nil
}
func isPaper(card scryfall.Card) bool {
var paper bool = false
for _, game := range card.Games {
if game == GAME_PAPER {
paper = true
}
}
return paper
}
func getCardPrintings(card scryfall.Card) []database.CardPrinting {
var printings []database.CardPrinting
if card.Foil {
printings = append(printings, database.CardPrinting{
Name: card.Name,
SetCode: card.Set,
IsFoil: true,
IsPromo: card.Promo,
CollectorNumber: card.CollectorNumber,
Language: card.Language,
})
}
if card.NonFoil {
printings = append(printings, database.CardPrinting{
Name: card.Name,
SetCode: card.Set,
IsFoil: false,
IsPromo: card.Promo,
CollectorNumber: card.CollectorNumber,
Language: card.Language,
})
}
return printings
}
func UpdateCards(db *sql.DB, bulkData scryfall.BulkData) error {
log.Println("Caching bulk cards file")
err := cacheBulkCardsFile(db, bulkData)
if err != nil {
return err
}
log.Println("Cached bulk cards file")
log.Println("Reading cached file")
cardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME)
if err != nil {
return err
}
log.Println("Read cached file")
log.Println("Unmarshaling JSON")
var cards []scryfall.Card
err = json.Unmarshal(cardsBytes, &cards)
if err != nil {
return err
}
log.Println("Unmarshaled JSON")
log.Println("INSERTing cards")
for _, card := range cards {
// We're only interested in paper cards, so skip cards or printings of a card which
// aren't available in paper
if !isPaper(card) {
continue
}
cardPrintings := getCardPrintings(card)
for _, cardPrinting := range cardPrintings {
err := database.InsertCardPrinting(db, cardPrinting)
// If we already have this card in the database, then we can just skip it
if ok, insertErr := sqlerr.Error(err); ok {
if insertErr == sqlerr.ErrDupeKey {
continue
}
}
}
if err != nil {
return err
}
}
log.Println("INSERTed cards")
return nil
}

View File

@ -4,23 +4,36 @@ import (
"fmt" "fmt"
"sevenkeys/database" "sevenkeys/database"
"sevenkeys/logic" "sevenkeys/logic"
"sevenkeys/logic/scryfall"
) )
func main() { func main() {
db := database.GetDatabaseFromConfig("config.json") db := database.GetDatabaseFromConfig("config.json")
fmt.Println("Checking for updates...") fmt.Println("Checking for updates...")
needsUpdate, err := logic.CheckForUpdates(db) bulkData, err := scryfall.GetBulkDataByType(scryfall.BulkDataTypeAllCards)
logic.Check(err)
needsUpdate, err := logic.CheckForUpdates(db, bulkData)
logic.Check(err) logic.Check(err)
if needsUpdate { if needsUpdate {
fmt.Println("Update required.") fmt.Println("Update required.")
if logic.ConfirmUpdate() { if logic.ConfirmUpdate() {
fmt.Println("User authorized update") fmt.Println("Running update...")
/*
err = logic.RunScryfallUpdate() logic.CreateCacheDirectories()
logic.Check(err)
*/ err = logic.UpdateSets(db)
logic.Check(err)
err = logic.UpdateCards(db, bulkData)
logic.Check(err)
} }
fmt.Println("Update finished.")
} else {
fmt.Println("No update required.")
} }
} }