TheMagician
/
TheMathemagicians
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor import/update code

This commit is contained in:
The Magician 2024-05-29 20:55:38 +01:00
parent 708bfdf90b
commit f84f19a0b7
13 changed files with 313 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
sevenkeys/database/cachetimestamp.go
View File

@ -1,6 +1,9 @@
package database
import "database/sql"
import (
"database/sql"
"time"
)
const CacheTypeAllCardsBulkData = "AllCardsBulkData"
@ -12,3 +15,23 @@ func GetCacheTimestampByType(db *sql.DB, cacheType string) (string, error) {
return timestamp, err
}
func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error {
query := `INSERT INTO CacheTimestamp (CacheType, Stamp)
VALUES (?, ?)
ON DUPLICATE KEY
UPDATE Stamp = ?;`
insertOrUpdate, err := db.Prepare(query)
defer insertOrUpdate.Close()
if err != nil {
return err
}
_, err = insertOrUpdate.Exec(cacheType, stamp, stamp)
if err != nil {
return err
}
return nil
}

39
sevenkeys/database/card.go Normal file
View File

@ -0,0 +1,39 @@
package database
import (
"database/sql"
)
type CardPrinting struct {
Id int
Name string
SetCode string
IsFoil bool
IsPromo bool
CollectorNumber string
Language string
}
func InsertCardPrinting(db *sql.DB, cardPrinting CardPrinting) error {
query := `INSERT INTO CardPrinting (
Name,
SetCode,
IsFoil,
IsPromo,
CollectorNumber,
Language)
VALUES (?, ?, ?, ?, ?, ?);`
insert, err := db.Prepare(query)
defer insert.Close()
if err != nil {
return err
}
_, err = insert.Exec(cardPrinting.Name, cardPrinting.SetCode, cardPrinting.IsFoil, cardPrinting.IsPromo, cardPrinting.CollectorNumber, cardPrinting.Language)
if err != nil {
return err
}
return nil
}

62
sevenkeys/database/operations/inserts.go
View File

@ -2,70 +2,8 @@ package operations
import (
"database/sql"
"sevenkeys/logic/scryfall/types"
"time"
)
func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error {
query := `INSERT INTO CacheTimestamp (CacheType, Stamp)
VALUES (?, ?)
ON DUPLICATE KEY
UPDATE Stamp = ?;`
insertOrUpdate, err := db.Prepare(query)
defer insertOrUpdate.Close()
if err != nil {
return err
}
_, err = insertOrUpdate.Exec(cacheType, stamp, stamp)
if err != nil {
return err
}
return nil
}
// TODO: There's really no need for this to be an "upsert", we can just ignore sets that we already have
func InsertOrUpdateSet(db *sql.DB, set types.Set) error {
query := `INSERT INTO ExpansionSet (SetCode, Name, CardCount, IconSvgUri)
VALUES (?, ?, ?, ?)
ON DUPLICATE KEY
Update Name = ?, CardCount = ?, IconSvgUri = ?;`
insertOrUpdate, err := db.Prepare(query)
defer insertOrUpdate.Close()
if err != nil {
return err
}
_, err = insertOrUpdate.Exec(set.Code, set.Name, set.CardCount, set.IconSvgUri, set.Name, set.CardCount, set.IconSvgUri)
if err != nil {
return err
}
return nil
}
func InsertCard(db *sql.DB, card types.Card) error {
query := `INSERT IGNORE INTO CardPrinting
(Id, Name, SetCode, HasFoil, HasNonFoil, IsReserved, IsRacist, IsPromo, CollectorNumber, Language)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`
insert, err := db.Prepare(query)
defer insert.Close()
if err != nil {
return err
}
_, err = insert.Exec(card.Id, card.Name, card.Set, card.Foil, card.NonFoil, card.Reserved, card.ContentWarning, card.Promo, card.CollectorNumber, card.Language)
if err != nil {
return err
}
return nil
}
func InsertCardStorageLocation(db *sql.DB, cardPrintingId string, isFoil bool, storageBox string, source string) error {
var lastPosition int
getLastPositionQuery := `SELECT Position FROM CardStorageLocation WHERE StorageBox = ? ORDER BY Position DESC LIMIT 1;`

23
sevenkeys/database/set.go Normal file
View File

@ -0,0 +1,23 @@
package database
import (
"database/sql"
"sevenkeys/logic/scryfall"
)
func InsertSet(db *sql.DB, set scryfall.Set) error {
query := `INSERT INTO ExpansionSet (SetCode, Name, CardCount, IconSvgUri) VALUES (?, ?, ?, ?);`
insert, err := db.Prepare(query)
defer insert.Close()
if err != nil {
return err
}
_, err = insert.Exec(set.Code, set.Name, set.CardCount, set.IconSvgUri)
if err != nil {
return err
}
return nil
}

25
sevenkeys/database/sql/createdb.sql
View File

@ -15,25 +15,22 @@ CREATE TABLE IF NOT EXISTS ExpansionSet (
);
CREATE TABLE IF NOT EXISTS CardPrinting (
Id VARCHAR(36) PRIMARY KEY, -- GUID
Id INT AUTO_INCREMENT PRIMARY KEY,
Name VARCHAR(150) NOT NULL,
SetCode VARCHAR(6) NOT NULL,
FOREIGN KEY (SetCode) REFERENCES ExpansionSet(SetCode),
HasFoil BOOLEAN NOT NULL,
HasNonFoil BOOLEAN NOT NULL,
IsReserved BOOLEAN NOT NULL,
IsRacist BOOLEAN NOT NULL,
IsFoil BOOLEAN NOT NULL,
IsPromo BOOLEAN NOT NULL,
CollectorNumber VARCHAR(10) NOT NULL,
Language VARCHAR(3) NOT NULL
);
CREATE TABLE IF NOT EXISTS CardStorageLocation (
Id INT AUTO_INCREMENT PRIMARY KEY,
CardPrintingId VARCHAR(36) NOT NULL,
FOREIGN KEY (CardPrintingId) REFERENCES CardPrinting(Id),
IsFoil BOOLEAN NOT NULL,
StorageBox VARCHAR(20) NOT NULL,
Source VARCHAR(100) NULL,
Position INT NOT NULL
);
-- CREATE TABLE IF NOT EXISTS CardStorageLocation (
-- Id INT AUTO_INCREMENT PRIMARY KEY,
-- CardPrintingId VARCHAR(36) NOT NULL,
-- FOREIGN KEY (CardPrintingId) REFERENCES CardPrinting(Id),
-- IsFoil BOOLEAN NOT NULL,
-- StorageBox VARCHAR(20) NOT NULL,
-- Source VARCHAR(100) NULL,
-- Position INT NOT NULL
-- );

1
sevenkeys/go.mod
View File

@ -6,6 +6,7 @@ require github.com/go-sql-driver/mysql v1.8.1
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0 // indirect
github.com/google/go-cmp v0.5.9 // indirect
gotest.tools/v3 v3.5.1 // indirect
)

2
sevenkeys/go.sum
View File

@ -1,5 +1,7 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0 h1:meiLwrW6ukHHehydhoDxVHdQKQe7TFgEpH0A0hHBAWs=
github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0/go.mod h1:ZH8V0509n2OSZLMYTMHzcy4hqUB+rG8ghK1zsP4i5gE=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=

175
sevenkeys/importer.go
View File

@ -1,175 +0,0 @@
package main
import (
"database/sql"
"encoding/json"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"sevenkeys/database/entities"
"sevenkeys/database/operations"
"sevenkeys/logic/scryfall/types"
"time"
)
const CACHE_DIR string = "cache"
const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/"
const SET_ICON_FILE_EXTENSION string = ".svg"
const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json"
const GAME_PAPER = "paper"
func check(err error) {
if err != nil {
log.Fatal(err)
}
}
func createCacheDirectories() error {
err := os.Mkdir(CACHE_DIR, os.ModePerm)
if err != nil && !os.IsExist(err) {
return err
}
err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm)
if err != nil && !os.IsExist(err) {
return err
}
return nil
}
func importSets(db *sql.DB, sets []types.Set) error {
for _, set := range sets {
// We're only interested in paper cards, so skip importing
// any sets that were only released in a video game
if set.Digital {
continue
}
log.Println("Importing " + set.Code)
err := operations.InsertOrUpdateSet(db, set)
if err != nil {
return err
}
log.Println("Downloading logo for " + set.Code)
response, err := http.Get(set.IconSvgUri)
defer response.Body.Close()
if err != nil {
return err
}
iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION
iconFile, err := os.Create(iconFilename)
if err != nil {
return err
}
io.Copy(iconFile, response.Body)
log.Println("Finished importing " + set.Code)
}
return nil
}
func cacheAllCardsFile(db *sql.DB, uri string, updatedAtTimestamp time.Time) error {
log.Printf("Downloading bulk card data...")
bulkCardsResponse, err := http.Get(uri)
if err != nil {
return err
}
log.Printf("Downloaded bulk card data.")
log.Printf("Writing card data to cache file...")
cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
if err != nil {
return err
}
defer bulkCardsResponse.Body.Close()
defer cacheFile.Close()
io.Copy(cacheFile, bulkCardsResponse.Body)
log.Printf("Cache file written.")
log.Printf("Recording timestamp...")
err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
if err != nil {
return err
}
log.Printf("Timestamp recorded.")
return nil
}
func importCards(db *sql.DB, cards []types.Card) error {
for _, card := range cards {
// We're only interested in paper cards, so skip cards or printings of a card which
// aren't available in paper
if !isPaper(card) {
continue
}
err := operations.InsertCard(db, card)
if err != nil {
return err
}
}
return nil
}
func isPaper(card types.Card) bool {
var paper bool = false
for _, game := range card.Games {
if game == GAME_PAPER {
paper = true
}
}
return paper
}
func main() {
log.Println("Creating cache directories...")
err := createCacheDirectories()
check(err)
log.Println("Created cache directories.")
log.Println("Downloading set data from Scryfall...")
sets, err := methods.GetSets()
check(err)
log.Println("Downloaded set data.")
log.Println("Importing set data...")
err = importSets(db, sets)
check(err)
log.Println("Imported sets.")
if updated {
log.Printf("Bulk data has been updated since last cache, redownloading.")
err = cacheAllCardsFile(db, allCardsBulkData.DownloadUri, updatedAtTimestamp)
check(err)
} else {
log.Printf("Bulk data has not been updated. Skipping download.")
}
log.Printf("Unmarsaling file into slice...")
allCardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME)
check(err)
var allCards []types.Card
err = json.Unmarshal(allCardsBytes, &allCards)
check(err)
log.Printf("Unmarshaled file.")
log.Printf("Importing card data into database...")
err = importCards(db, allCards)
check(err)
log.Printf("Imported card data.")
}

29
sevenkeys/logic/scryfall/bulkdata.go
View File

@ -5,19 +5,21 @@ import (
"errors"
"io"
"net/http"
"time"
)
type BulkData struct {
Id string `json:"id"`
Uri string `json:"uri"`
Type string `json:"type"`
Name string `json:"name"`
Description string `json:"description"`
DownloadUri string `json:"download_uri"`
UpdatedAt string `json:"updated_at"`
Size int `json:"size"`
ContentType string `json:"content_type"`
ContentEncoding string `json:"content_encoding"`
Id string `json:"id"`
Uri string `json:"uri"`
Type string `json:"type"`
Name string `json:"name"`
Description string `json:"description"`
DownloadUri string `json:"download_uri"`
UpdatedAt string `json:"updated_at"`
UpdatedAtTime time.Time `json:"ignore"`
Size int `json:"size"`
ContentType string `json:"content_type"`
ContentEncoding string `json:"content_encoding"`
}
const BULK_DATA_URI = "https://api.scryfall.com/bulk-data"
@ -50,5 +52,12 @@ func GetBulkDataByType(bulkDataType string) (BulkData, error) {
return BulkData{}, err
}
bulkData.UpdatedAtTime, err = time.Parse(ScryfallTimestampFormat, bulkData.UpdatedAt)
if err != nil {
return BulkData{}, err
}
// Round to the nearest second; this is so that comparison with the timestamp stored in the database works as intended
bulkData.UpdatedAtTime = bulkData.UpdatedAtTime.Truncate(time.Second)
return bulkData, nil
}

3
sevenkeys/logic/scryfall/card.go
View File

@ -1,14 +1,11 @@
package scryfall
type Card struct {
Id string `json:"id"` // GUID
Name string `json:"name"`
Set string `json:"set"`
Games []string `json:"games"`
Foil bool `json:"foil"`
NonFoil bool `json:"nonfoil"`
Reserved bool `json:"reserved"`
ContentWarning bool `json:"content_warning,omitempty"`
Promo bool `json:"promo"`
CollectorNumber string `json:"collector_number"`
Language string `json:"lang"`

2
sevenkeys/logic/scryfall/sets.go
View File

@ -23,7 +23,7 @@ type Set struct {
const SETS_API_URL string = "https://api.scryfall.com/sets"
func GetSets() ([]Set, error) {
func GetAllSets() ([]Set, error) {
response, err := http.Get(SETS_API_URL)
if err != nil {
return []Set{}, nil

187
sevenkeys/logic/update.go
View File

@ -2,14 +2,27 @@ package logic
import (
"database/sql"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"sevenkeys/database"
"sevenkeys/logic/scryfall"
"strings"
"time"
sqlerr "github.com/go-mysql/errors"
)
func CheckForUpdates(db *sql.DB) (bool, error) {
const GAME_PAPER = "paper"
func CheckForUpdates(db *sql.DB, bulkData scryfall.BulkData) (bool, error) {
// TODO: We also want to update if:
// - there is no cached version of the all-cards.json file or it is empty
// - The set icon files are missing
cachedFileTimestampStr, err := database.GetCacheTimestampByType(db, database.CacheTypeAllCardsBulkData)
if err == sql.ErrNoRows {
return true, nil
@ -19,18 +32,7 @@ func CheckForUpdates(db *sql.DB) (bool, error) {
cachedFileTimestamp, err := time.Parse("2006-01-02 15:04:05", cachedFileTimestampStr)
allCardsBulkData, err := scryfall.GetBulkDataByType(scryfall.BulkDataTypeAllCards)
if err != nil {
return false, err
}
bulkCardsUpdatedTimestamp, err := time.Parse(scryfall.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
if err != nil {
return false, err
}
bulkCardsUpdatedTimestamp = bulkCardsUpdatedTimestamp.Truncate(time.Second)
return bulkCardsUpdatedTimestamp.After(cachedFileTimestamp), nil
return bulkData.UpdatedAtTime.After(cachedFileTimestamp), nil
}
func ConfirmUpdate() bool {
@ -40,3 +42,162 @@ func ConfirmUpdate() bool {
return strings.ToUpper(response) == "Y"
}
func UpdateSets(db *sql.DB) error {
sets, err := scryfall.GetAllSets()
if err != nil {
return err
}
for _, set := range sets {
// We're only interested in paper cards, so skip importing
// any sets that were only released in a video game
if set.Digital {
continue
}
err := database.InsertSet(db, set)
// If we already have this set in the database, then we can just skip it
if ok, insertErr := sqlerr.Error(err); ok {
if insertErr == sqlerr.ErrDupeKey {
continue
}
}
if err != nil {
return err
}
response, err := http.Get(set.IconSvgUri)
defer response.Body.Close()
if err != nil {
return nil
}
iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION
iconFile, err := os.Create(iconFilename)
if err != nil {
return err
}
io.Copy(iconFile, response.Body)
}
fmt.Println("Sets updated.")
return nil
}
func cacheBulkCardsFile(db *sql.DB, bulkData scryfall.BulkData) error {
bulkCardsResponse, err := http.Get(bulkData.DownloadUri)
defer bulkCardsResponse.Body.Close()
if err != nil {
return err
}
cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
defer cacheFile.Close()
if err != nil {
return err
}
io.Copy(cacheFile, bulkCardsResponse.Body)
err = database.InsertOrUpdateCacheTimestampByType(db, database.CacheTypeAllCardsBulkData, bulkData.UpdatedAtTime)
if err != nil {
return err
}
return nil
}
func isPaper(card scryfall.Card) bool {
var paper bool = false
for _, game := range card.Games {
if game == GAME_PAPER {
paper = true
}
}
return paper
}
func getCardPrintings(card scryfall.Card) []database.CardPrinting {
var printings []database.CardPrinting
if card.Foil {
printings = append(printings, database.CardPrinting{
Name: card.Name,
SetCode: card.Set,
IsFoil: true,
IsPromo: card.Promo,
CollectorNumber: card.CollectorNumber,
Language: card.Language,
})
}
if card.NonFoil {
printings = append(printings, database.CardPrinting{
Name: card.Name,
SetCode: card.Set,
IsFoil: false,
IsPromo: card.Promo,
CollectorNumber: card.CollectorNumber,
Language: card.Language,
})
}
return printings
}
func UpdateCards(db *sql.DB, bulkData scryfall.BulkData) error {
log.Println("Caching bulk cards file")
err := cacheBulkCardsFile(db, bulkData)
if err != nil {
return err
}
log.Println("Cached bulk cards file")
log.Println("Reading cached file")
cardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME)
if err != nil {
return err
}
log.Println("Read cached file")
log.Println("Unmarshaling JSON")
var cards []scryfall.Card
err = json.Unmarshal(cardsBytes, &cards)
if err != nil {
return err
}
log.Println("Unmarshaled JSON")
log.Println("INSERTing cards")
for _, card := range cards {
// We're only interested in paper cards, so skip cards or printings of a card which
// aren't available in paper
if !isPaper(card) {
continue
}
cardPrintings := getCardPrintings(card)
for _, cardPrinting := range cardPrintings {
err := database.InsertCardPrinting(db, cardPrinting)
// If we already have this card in the database, then we can just skip it
if ok, insertErr := sqlerr.Error(err); ok {
if insertErr == sqlerr.ErrDupeKey {
continue
}
}
}
if err != nil {
return err
}
}
log.Println("INSERTed cards")
return nil
}

25
sevenkeys/main.go
View File

@ -4,23 +4,36 @@ import (
"fmt"
"sevenkeys/database"
"sevenkeys/logic"
"sevenkeys/logic/scryfall"
)
func main() {
db := database.GetDatabaseFromConfig("config.json")
fmt.Println("Checking for updates...")
needsUpdate, err := logic.CheckForUpdates(db)
bulkData, err := scryfall.GetBulkDataByType(scryfall.BulkDataTypeAllCards)
logic.Check(err)
needsUpdate, err := logic.CheckForUpdates(db, bulkData)
logic.Check(err)
if needsUpdate {
fmt.Println("Update required.")
if logic.ConfirmUpdate() {
fmt.Println("User authorized update")
/*
err = logic.RunScryfallUpdate()
logic.Check(err)
*/
fmt.Println("Running update...")
logic.CreateCacheDirectories()
err = logic.UpdateSets(db)
logic.Check(err)
err = logic.UpdateCards(db, bulkData)
logic.Check(err)
}
fmt.Println("Update finished.")
} else {
fmt.Println("No update required.")
}
}