diff --git a/sevenkeys/database/cachetimestamp.go b/sevenkeys/database/cachetimestamp.go index 7910059..ffb5447 100644 --- a/sevenkeys/database/cachetimestamp.go +++ b/sevenkeys/database/cachetimestamp.go @@ -1,6 +1,9 @@ package database -import "database/sql" +import ( + "database/sql" + "time" +) const CacheTypeAllCardsBulkData = "AllCardsBulkData" @@ -12,3 +15,23 @@ func GetCacheTimestampByType(db *sql.DB, cacheType string) (string, error) { return timestamp, err } + +func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error { + query := `INSERT INTO CacheTimestamp (CacheType, Stamp) + VALUES (?, ?) + ON DUPLICATE KEY + UPDATE Stamp = ?;` + + insertOrUpdate, err := db.Prepare(query) + defer insertOrUpdate.Close() + if err != nil { + return err + } + + _, err = insertOrUpdate.Exec(cacheType, stamp, stamp) + if err != nil { + return err + } + + return nil +} diff --git a/sevenkeys/database/card.go b/sevenkeys/database/card.go new file mode 100644 index 0000000..dbf1305 --- /dev/null +++ b/sevenkeys/database/card.go @@ -0,0 +1,39 @@ +package database + +import ( + "database/sql" +) + +type CardPrinting struct { + Id int + Name string + SetCode string + IsFoil bool + IsPromo bool + CollectorNumber string + Language string +} + +func InsertCardPrinting(db *sql.DB, cardPrinting CardPrinting) error { + query := `INSERT INTO CardPrinting ( + Name, + SetCode, + IsFoil, + IsPromo, + CollectorNumber, + Language) + VALUES (?, ?, ?, ?, ?, ?);` + + insert, err := db.Prepare(query) + defer insert.Close() + if err != nil { + return err + } + + _, err = insert.Exec(cardPrinting.Name, cardPrinting.SetCode, cardPrinting.IsFoil, cardPrinting.IsPromo, cardPrinting.CollectorNumber, cardPrinting.Language) + if err != nil { + return err + } + + return nil +} diff --git a/sevenkeys/database/operations/inserts.go b/sevenkeys/database/operations/inserts.go index badb025..11683c5 100644 --- a/sevenkeys/database/operations/inserts.go +++ b/sevenkeys/database/operations/inserts.go @@ -2,70 +2,8 @@ package operations import ( "database/sql" - "sevenkeys/logic/scryfall/types" - "time" ) -func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error { - query := `INSERT INTO CacheTimestamp (CacheType, Stamp) - VALUES (?, ?) - ON DUPLICATE KEY - UPDATE Stamp = ?;` - - insertOrUpdate, err := db.Prepare(query) - defer insertOrUpdate.Close() - if err != nil { - return err - } - - _, err = insertOrUpdate.Exec(cacheType, stamp, stamp) - if err != nil { - return err - } - - return nil -} - -// TODO: There's really no need for this to be an "upsert", we can just ignore sets that we already have -func InsertOrUpdateSet(db *sql.DB, set types.Set) error { - query := `INSERT INTO ExpansionSet (SetCode, Name, CardCount, IconSvgUri) - VALUES (?, ?, ?, ?) - ON DUPLICATE KEY - Update Name = ?, CardCount = ?, IconSvgUri = ?;` - - insertOrUpdate, err := db.Prepare(query) - defer insertOrUpdate.Close() - if err != nil { - return err - } - - _, err = insertOrUpdate.Exec(set.Code, set.Name, set.CardCount, set.IconSvgUri, set.Name, set.CardCount, set.IconSvgUri) - if err != nil { - return err - } - - return nil -} - -func InsertCard(db *sql.DB, card types.Card) error { - query := `INSERT IGNORE INTO CardPrinting - (Id, Name, SetCode, HasFoil, HasNonFoil, IsReserved, IsRacist, IsPromo, CollectorNumber, Language) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);` - - insert, err := db.Prepare(query) - defer insert.Close() - if err != nil { - return err - } - - _, err = insert.Exec(card.Id, card.Name, card.Set, card.Foil, card.NonFoil, card.Reserved, card.ContentWarning, card.Promo, card.CollectorNumber, card.Language) - if err != nil { - return err - } - - return nil -} - func InsertCardStorageLocation(db *sql.DB, cardPrintingId string, isFoil bool, storageBox string, source string) error { var lastPosition int getLastPositionQuery := `SELECT Position FROM CardStorageLocation WHERE StorageBox = ? ORDER BY Position DESC LIMIT 1;` diff --git a/sevenkeys/database/set.go b/sevenkeys/database/set.go new file mode 100644 index 0000000..d27a619 --- /dev/null +++ b/sevenkeys/database/set.go @@ -0,0 +1,23 @@ +package database + +import ( + "database/sql" + "sevenkeys/logic/scryfall" +) + +func InsertSet(db *sql.DB, set scryfall.Set) error { + query := `INSERT INTO ExpansionSet (SetCode, Name, CardCount, IconSvgUri) VALUES (?, ?, ?, ?);` + + insert, err := db.Prepare(query) + defer insert.Close() + if err != nil { + return err + } + + _, err = insert.Exec(set.Code, set.Name, set.CardCount, set.IconSvgUri) + if err != nil { + return err + } + + return nil +} diff --git a/sevenkeys/database/sql/createdb.sql b/sevenkeys/database/sql/createdb.sql index 21b8d81..b319e66 100644 --- a/sevenkeys/database/sql/createdb.sql +++ b/sevenkeys/database/sql/createdb.sql @@ -15,25 +15,22 @@ CREATE TABLE IF NOT EXISTS ExpansionSet ( ); CREATE TABLE IF NOT EXISTS CardPrinting ( - Id VARCHAR(36) PRIMARY KEY, -- GUID + Id INT AUTO_INCREMENT PRIMARY KEY, Name VARCHAR(150) NOT NULL, SetCode VARCHAR(6) NOT NULL, FOREIGN KEY (SetCode) REFERENCES ExpansionSet(SetCode), - HasFoil BOOLEAN NOT NULL, - HasNonFoil BOOLEAN NOT NULL, - IsReserved BOOLEAN NOT NULL, - IsRacist BOOLEAN NOT NULL, + IsFoil BOOLEAN NOT NULL, IsPromo BOOLEAN NOT NULL, CollectorNumber VARCHAR(10) NOT NULL, Language VARCHAR(3) NOT NULL ); -CREATE TABLE IF NOT EXISTS CardStorageLocation ( - Id INT AUTO_INCREMENT PRIMARY KEY, - CardPrintingId VARCHAR(36) NOT NULL, - FOREIGN KEY (CardPrintingId) REFERENCES CardPrinting(Id), - IsFoil BOOLEAN NOT NULL, - StorageBox VARCHAR(20) NOT NULL, - Source VARCHAR(100) NULL, - Position INT NOT NULL -); +-- CREATE TABLE IF NOT EXISTS CardStorageLocation ( + -- Id INT AUTO_INCREMENT PRIMARY KEY, + -- CardPrintingId VARCHAR(36) NOT NULL, + -- FOREIGN KEY (CardPrintingId) REFERENCES CardPrinting(Id), + -- IsFoil BOOLEAN NOT NULL, + -- StorageBox VARCHAR(20) NOT NULL, + -- Source VARCHAR(100) NULL, + -- Position INT NOT NULL +-- ); diff --git a/sevenkeys/go.mod b/sevenkeys/go.mod index 575ba71..ec3bf54 100644 --- a/sevenkeys/go.mod +++ b/sevenkeys/go.mod @@ -6,6 +6,7 @@ require github.com/go-sql-driver/mysql v1.8.1 require ( filippo.io/edwards25519 v1.1.0 // indirect + github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0 // indirect github.com/google/go-cmp v0.5.9 // indirect gotest.tools/v3 v3.5.1 // indirect ) diff --git a/sevenkeys/go.sum b/sevenkeys/go.sum index 33a06fb..e99fdf1 100644 --- a/sevenkeys/go.sum +++ b/sevenkeys/go.sum @@ -1,5 +1,7 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0 h1:meiLwrW6ukHHehydhoDxVHdQKQe7TFgEpH0A0hHBAWs= +github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0/go.mod h1:ZH8V0509n2OSZLMYTMHzcy4hqUB+rG8ghK1zsP4i5gE= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= diff --git a/sevenkeys/importer.go b/sevenkeys/importer.go deleted file mode 100644 index a01f288..0000000 --- a/sevenkeys/importer.go +++ /dev/null @@ -1,175 +0,0 @@ -package main - -import ( - "database/sql" - "encoding/json" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "sevenkeys/database/entities" - "sevenkeys/database/operations" - "sevenkeys/logic/scryfall/types" - "time" -) - -const CACHE_DIR string = "cache" - -const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/" -const SET_ICON_FILE_EXTENSION string = ".svg" - -const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json" - -const GAME_PAPER = "paper" - -func check(err error) { - if err != nil { - log.Fatal(err) - } -} - -func createCacheDirectories() error { - err := os.Mkdir(CACHE_DIR, os.ModePerm) - if err != nil && !os.IsExist(err) { - return err - } - - err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm) - if err != nil && !os.IsExist(err) { - return err - } - - return nil -} - -func importSets(db *sql.DB, sets []types.Set) error { - for _, set := range sets { - // We're only interested in paper cards, so skip importing - // any sets that were only released in a video game - if set.Digital { - continue - } - - log.Println("Importing " + set.Code) - err := operations.InsertOrUpdateSet(db, set) - if err != nil { - return err - } - - log.Println("Downloading logo for " + set.Code) - response, err := http.Get(set.IconSvgUri) - defer response.Body.Close() - if err != nil { - return err - } - - iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION - iconFile, err := os.Create(iconFilename) - if err != nil { - return err - } - - io.Copy(iconFile, response.Body) - log.Println("Finished importing " + set.Code) - } - - return nil -} - -func cacheAllCardsFile(db *sql.DB, uri string, updatedAtTimestamp time.Time) error { - log.Printf("Downloading bulk card data...") - bulkCardsResponse, err := http.Get(uri) - if err != nil { - return err - } - log.Printf("Downloaded bulk card data.") - - log.Printf("Writing card data to cache file...") - cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME) - if err != nil { - return err - } - - defer bulkCardsResponse.Body.Close() - defer cacheFile.Close() - io.Copy(cacheFile, bulkCardsResponse.Body) - log.Printf("Cache file written.") - - log.Printf("Recording timestamp...") - err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp) - if err != nil { - return err - } - log.Printf("Timestamp recorded.") - - return nil -} - -func importCards(db *sql.DB, cards []types.Card) error { - for _, card := range cards { - // We're only interested in paper cards, so skip cards or printings of a card which - // aren't available in paper - if !isPaper(card) { - continue - } - - err := operations.InsertCard(db, card) - if err != nil { - return err - } - } - - return nil -} - -func isPaper(card types.Card) bool { - var paper bool = false - - for _, game := range card.Games { - if game == GAME_PAPER { - paper = true - } - } - - return paper -} - -func main() { - log.Println("Creating cache directories...") - err := createCacheDirectories() - check(err) - log.Println("Created cache directories.") - - log.Println("Downloading set data from Scryfall...") - sets, err := methods.GetSets() - check(err) - log.Println("Downloaded set data.") - - log.Println("Importing set data...") - err = importSets(db, sets) - check(err) - log.Println("Imported sets.") - - if updated { - log.Printf("Bulk data has been updated since last cache, redownloading.") - err = cacheAllCardsFile(db, allCardsBulkData.DownloadUri, updatedAtTimestamp) - check(err) - } else { - log.Printf("Bulk data has not been updated. Skipping download.") - } - - log.Printf("Unmarsaling file into slice...") - allCardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME) - check(err) - - var allCards []types.Card - err = json.Unmarshal(allCardsBytes, &allCards) - check(err) - log.Printf("Unmarshaled file.") - - log.Printf("Importing card data into database...") - err = importCards(db, allCards) - check(err) - log.Printf("Imported card data.") -} diff --git a/sevenkeys/logic/scryfall/bulkdata.go b/sevenkeys/logic/scryfall/bulkdata.go index fb80be7..0b5e09a 100644 --- a/sevenkeys/logic/scryfall/bulkdata.go +++ b/sevenkeys/logic/scryfall/bulkdata.go @@ -5,19 +5,21 @@ import ( "errors" "io" "net/http" + "time" ) type BulkData struct { - Id string `json:"id"` - Uri string `json:"uri"` - Type string `json:"type"` - Name string `json:"name"` - Description string `json:"description"` - DownloadUri string `json:"download_uri"` - UpdatedAt string `json:"updated_at"` - Size int `json:"size"` - ContentType string `json:"content_type"` - ContentEncoding string `json:"content_encoding"` + Id string `json:"id"` + Uri string `json:"uri"` + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + DownloadUri string `json:"download_uri"` + UpdatedAt string `json:"updated_at"` + UpdatedAtTime time.Time `json:"ignore"` + Size int `json:"size"` + ContentType string `json:"content_type"` + ContentEncoding string `json:"content_encoding"` } const BULK_DATA_URI = "https://api.scryfall.com/bulk-data" @@ -50,5 +52,12 @@ func GetBulkDataByType(bulkDataType string) (BulkData, error) { return BulkData{}, err } + bulkData.UpdatedAtTime, err = time.Parse(ScryfallTimestampFormat, bulkData.UpdatedAt) + if err != nil { + return BulkData{}, err + } + // Round to the nearest second; this is so that comparison with the timestamp stored in the database works as intended + bulkData.UpdatedAtTime = bulkData.UpdatedAtTime.Truncate(time.Second) + return bulkData, nil } diff --git a/sevenkeys/logic/scryfall/card.go b/sevenkeys/logic/scryfall/card.go index bf226cb..0f8b588 100644 --- a/sevenkeys/logic/scryfall/card.go +++ b/sevenkeys/logic/scryfall/card.go @@ -1,14 +1,11 @@ package scryfall type Card struct { - Id string `json:"id"` // GUID Name string `json:"name"` Set string `json:"set"` Games []string `json:"games"` Foil bool `json:"foil"` NonFoil bool `json:"nonfoil"` - Reserved bool `json:"reserved"` - ContentWarning bool `json:"content_warning,omitempty"` Promo bool `json:"promo"` CollectorNumber string `json:"collector_number"` Language string `json:"lang"` diff --git a/sevenkeys/logic/scryfall/sets.go b/sevenkeys/logic/scryfall/sets.go index 1702d20..d734f4f 100644 --- a/sevenkeys/logic/scryfall/sets.go +++ b/sevenkeys/logic/scryfall/sets.go @@ -23,7 +23,7 @@ type Set struct { const SETS_API_URL string = "https://api.scryfall.com/sets" -func GetSets() ([]Set, error) { +func GetAllSets() ([]Set, error) { response, err := http.Get(SETS_API_URL) if err != nil { return []Set{}, nil diff --git a/sevenkeys/logic/update.go b/sevenkeys/logic/update.go index 2ce07b8..3e6f923 100644 --- a/sevenkeys/logic/update.go +++ b/sevenkeys/logic/update.go @@ -2,14 +2,27 @@ package logic import ( "database/sql" + "encoding/json" "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" "sevenkeys/database" "sevenkeys/logic/scryfall" "strings" "time" + + sqlerr "github.com/go-mysql/errors" ) -func CheckForUpdates(db *sql.DB) (bool, error) { +const GAME_PAPER = "paper" + +func CheckForUpdates(db *sql.DB, bulkData scryfall.BulkData) (bool, error) { + // TODO: We also want to update if: + // - there is no cached version of the all-cards.json file or it is empty + // - The set icon files are missing cachedFileTimestampStr, err := database.GetCacheTimestampByType(db, database.CacheTypeAllCardsBulkData) if err == sql.ErrNoRows { return true, nil @@ -19,18 +32,7 @@ func CheckForUpdates(db *sql.DB) (bool, error) { cachedFileTimestamp, err := time.Parse("2006-01-02 15:04:05", cachedFileTimestampStr) - allCardsBulkData, err := scryfall.GetBulkDataByType(scryfall.BulkDataTypeAllCards) - if err != nil { - return false, err - } - - bulkCardsUpdatedTimestamp, err := time.Parse(scryfall.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt) - if err != nil { - return false, err - } - bulkCardsUpdatedTimestamp = bulkCardsUpdatedTimestamp.Truncate(time.Second) - - return bulkCardsUpdatedTimestamp.After(cachedFileTimestamp), nil + return bulkData.UpdatedAtTime.After(cachedFileTimestamp), nil } func ConfirmUpdate() bool { @@ -40,3 +42,162 @@ func ConfirmUpdate() bool { return strings.ToUpper(response) == "Y" } + +func UpdateSets(db *sql.DB) error { + sets, err := scryfall.GetAllSets() + if err != nil { + return err + } + + for _, set := range sets { + // We're only interested in paper cards, so skip importing + // any sets that were only released in a video game + if set.Digital { + continue + } + + err := database.InsertSet(db, set) + // If we already have this set in the database, then we can just skip it + if ok, insertErr := sqlerr.Error(err); ok { + if insertErr == sqlerr.ErrDupeKey { + continue + } + } + + if err != nil { + return err + } + + response, err := http.Get(set.IconSvgUri) + defer response.Body.Close() + if err != nil { + return nil + } + + iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION + iconFile, err := os.Create(iconFilename) + if err != nil { + return err + } + + io.Copy(iconFile, response.Body) + } + + fmt.Println("Sets updated.") + return nil +} + +func cacheBulkCardsFile(db *sql.DB, bulkData scryfall.BulkData) error { + bulkCardsResponse, err := http.Get(bulkData.DownloadUri) + defer bulkCardsResponse.Body.Close() + if err != nil { + return err + } + + cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME) + defer cacheFile.Close() + if err != nil { + return err + } + + io.Copy(cacheFile, bulkCardsResponse.Body) + + err = database.InsertOrUpdateCacheTimestampByType(db, database.CacheTypeAllCardsBulkData, bulkData.UpdatedAtTime) + if err != nil { + return err + } + + return nil +} + +func isPaper(card scryfall.Card) bool { + var paper bool = false + + for _, game := range card.Games { + if game == GAME_PAPER { + paper = true + } + } + + return paper +} + +func getCardPrintings(card scryfall.Card) []database.CardPrinting { + var printings []database.CardPrinting + + if card.Foil { + printings = append(printings, database.CardPrinting{ + Name: card.Name, + SetCode: card.Set, + IsFoil: true, + IsPromo: card.Promo, + CollectorNumber: card.CollectorNumber, + Language: card.Language, + }) + } + + if card.NonFoil { + printings = append(printings, database.CardPrinting{ + Name: card.Name, + SetCode: card.Set, + IsFoil: false, + IsPromo: card.Promo, + CollectorNumber: card.CollectorNumber, + Language: card.Language, + }) + } + + return printings +} + +func UpdateCards(db *sql.DB, bulkData scryfall.BulkData) error { + log.Println("Caching bulk cards file") + err := cacheBulkCardsFile(db, bulkData) + if err != nil { + return err + } + log.Println("Cached bulk cards file") + + log.Println("Reading cached file") + cardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME) + if err != nil { + return err + } + log.Println("Read cached file") + + log.Println("Unmarshaling JSON") + var cards []scryfall.Card + err = json.Unmarshal(cardsBytes, &cards) + if err != nil { + return err + } + log.Println("Unmarshaled JSON") + + log.Println("INSERTing cards") + for _, card := range cards { + // We're only interested in paper cards, so skip cards or printings of a card which + // aren't available in paper + if !isPaper(card) { + continue + } + + cardPrintings := getCardPrintings(card) + + for _, cardPrinting := range cardPrintings { + err := database.InsertCardPrinting(db, cardPrinting) + // If we already have this card in the database, then we can just skip it + if ok, insertErr := sqlerr.Error(err); ok { + if insertErr == sqlerr.ErrDupeKey { + continue + } + } + } + + if err != nil { + return err + } + } + log.Println("INSERTed cards") + + return nil +} diff --git a/sevenkeys/main.go b/sevenkeys/main.go index 54d79f7..3b496b8 100644 --- a/sevenkeys/main.go +++ b/sevenkeys/main.go @@ -4,23 +4,36 @@ import ( "fmt" "sevenkeys/database" "sevenkeys/logic" + "sevenkeys/logic/scryfall" ) func main() { db := database.GetDatabaseFromConfig("config.json") fmt.Println("Checking for updates...") - needsUpdate, err := logic.CheckForUpdates(db) + bulkData, err := scryfall.GetBulkDataByType(scryfall.BulkDataTypeAllCards) + logic.Check(err) + + needsUpdate, err := logic.CheckForUpdates(db, bulkData) logic.Check(err) if needsUpdate { fmt.Println("Update required.") + if logic.ConfirmUpdate() { - fmt.Println("User authorized update") - /* - err = logic.RunScryfallUpdate() - logic.Check(err) - */ + fmt.Println("Running update...") + + logic.CreateCacheDirectories() + + err = logic.UpdateSets(db) + logic.Check(err) + + err = logic.UpdateCards(db, bulkData) + logic.Check(err) } + + fmt.Println("Update finished.") + } else { + fmt.Println("No update required.") } }