diff --git a/sevenkeys/logic/cache.go b/sevenkeys/logic/cache.go deleted file mode 100644 index 6b63655..0000000 --- a/sevenkeys/logic/cache.go +++ /dev/null @@ -1,22 +0,0 @@ -package logic - -import "os" - -const CACHE_DIR string = "cache" -const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/" -const SET_ICON_FILE_EXTENSION string = ".svg" -const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json" - -func CreateCacheDirectories() error { - err := os.Mkdir(CACHE_DIR, os.ModePerm) - if err != nil && !os.IsExist(err) { - return err - } - - err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm) - if err != nil && !os.IsExist(err) { - return err - } - - return nil -} diff --git a/sevenkeys/logic/update.go b/sevenkeys/logic/update.go deleted file mode 100644 index 1c605b2..0000000 --- a/sevenkeys/logic/update.go +++ /dev/null @@ -1,197 +0,0 @@ -package logic - -import ( - "database/sql" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "sevenkeys/database" - "sevenkeys/logic/scryfall" - "time" - - sqlerr "github.com/go-mysql/errors" -) - -const GAME_PAPER = "paper" - -func CheckForUpdates(db *sql.DB, bulkData scryfall.BulkData) (bool, error) { - // TODO: We also want to update if: - // - there is no cached version of the all-cards.json file or it is empty - // - The set icon files are missing - cachedFileTimestampStr, err := database.GetCacheTimestampByType(db, database.CacheTypeAllCardsBulkData) - if err == sql.ErrNoRows { - return true, nil - } else if err != nil { - return false, err - } - - cachedFileTimestamp, err := time.Parse("2006-01-02 15:04:05", cachedFileTimestampStr) - - return bulkData.UpdatedAtTime.After(cachedFileTimestamp), nil -} - -func UpdateSets(db *sql.DB) error { - sets, err := scryfall.GetAllSets() - if err != nil { - return err - } - - for _, set := range sets { - // We're only interested in paper cards, so skip importing - // any sets that were only released in a video game - if set.Digital { - continue - } - - err := database.InsertSet(db, set) - // If we already have this set in the database, then we can just skip it - if ok, insertErr := sqlerr.Error(err); ok { - if insertErr == sqlerr.ErrDupeKey { - continue - } - } - - if err != nil { - return err - } - - response, err := http.Get(set.IconSvgUri) - defer response.Body.Close() - if err != nil { - return nil - } - - iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION - iconFile, err := os.Create(iconFilename) - if err != nil { - return err - } - - io.Copy(iconFile, response.Body) - } - - fmt.Println("Sets updated.") - return nil -} - -func cacheBulkCardsFile(db *sql.DB, bulkData scryfall.BulkData) error { - bulkCardsResponse, err := http.Get(bulkData.DownloadUri) - defer bulkCardsResponse.Body.Close() - if err != nil { - return err - } - - cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME) - defer cacheFile.Close() - if err != nil { - return err - } - - io.Copy(cacheFile, bulkCardsResponse.Body) - - err = database.InsertOrUpdateCacheTimestampByType(db, database.CacheTypeAllCardsBulkData, bulkData.UpdatedAtTime) - if err != nil { - return err - } - - return nil -} - -func isPaper(card scryfall.Card) bool { - var paper bool = false - - for _, game := range card.Games { - if game == GAME_PAPER { - paper = true - } - } - - return paper -} - -func getCardPrintings(card scryfall.Card) []database.CardPrinting { - var printings []database.CardPrinting - - if card.Foil { - printings = append(printings, database.CardPrinting{ - Id: card.Id + "f", - Name: card.Name, - SetCode: card.Set, - IsFoil: true, - IsPromo: card.Promo, - CollectorNumber: card.CollectorNumber, - ImageUrl: card.ImageUris["png"], - Language: card.Language, - }) - } - - if card.NonFoil { - printings = append(printings, database.CardPrinting{ - Id: card.Id + "n", - Name: card.Name, - SetCode: card.Set, - IsFoil: false, - IsPromo: card.Promo, - CollectorNumber: card.CollectorNumber, - Language: card.Language, - }) - } - - return printings -} - -func UpdateCards(db *sql.DB, bulkData scryfall.BulkData) error { - log.Println("Caching bulk cards file") - err := cacheBulkCardsFile(db, bulkData) - if err != nil { - return err - } - log.Println("Cached bulk cards file") - - log.Println("Reading cached file") - cardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME) - if err != nil { - return err - } - log.Println("Read cached file") - - log.Println("Unmarshaling JSON") - var cards []scryfall.Card - err = json.Unmarshal(cardsBytes, &cards) - if err != nil { - return err - } - log.Println("Unmarshaled JSON") - - log.Println("INSERTing cards") - for _, card := range cards { - // We're only interested in paper cards, so skip cards or printings of a card which - // aren't available in paper - if !isPaper(card) { - continue - } - - cardPrintings := getCardPrintings(card) - - for _, cardPrinting := range cardPrintings { - err := database.InsertCardPrinting(db, cardPrinting) - // If we already have this card in the database, then we can just skip it - if ok, insertErr := sqlerr.Error(err); ok { - if insertErr == sqlerr.ErrDupeKey { - continue - } - } - } - - if err != nil { - return err - } - } - log.Println("INSERTed cards") - - return nil -} diff --git a/sevenkeys/update/scryfall.go b/sevenkeys/update/scryfall.go index 96da144..0c0c68b 100644 --- a/sevenkeys/update/scryfall.go +++ b/sevenkeys/update/scryfall.go @@ -2,31 +2,215 @@ package update import ( "database/sql" - "fmt" + "encoding/json" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "sevenkeys/database" "sevenkeys/logic" "sevenkeys/logic/scryfall" + "time" ) +func scryfallLog(msg string) { + log.Printf("scryfall: %s\n", msg) +} + +const CACHE_DIR string = "cache" +const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/" +const SET_ICON_FILE_EXTENSION string = ".svg" +const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json" +const GAME_PAPER = "paper" + +func createCacheDirectories() error { + err := os.Mkdir(CACHE_DIR, os.ModePerm) + if err != nil && !os.IsExist(err) { + return err + } + + err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm) + if err != nil && !os.IsExist(err) { + return err + } + + return nil +} + +func checkScryfallUpdates(db *sql.DB, bulkData scryfall.BulkData) (bool, error) { + cachedFileTimestampStr, err := database.GetCacheTimestampByType(db, database.CacheTypeAllCardsBulkData) + if err == sql.ErrNoRows { + return true, nil + } else if err != nil { + return false, err + } + + cachedFileTimestamp, err := time.Parse("2006-01-02 15:04:05", cachedFileTimestampStr) + + return bulkData.UpdatedAtTime.After(cachedFileTimestamp), nil +} + +func updateSets(db *sql.DB) error { + sets, err := scryfall.GetAllSets() + if err != nil { + return err + } + + for _, set := range sets { + // We're only interested in paper cards, so skip importing + // any sets that were only released in a video game + if set.Digital { + continue + } + + err := database.InsertSet(db, set) + // If we already have this set in the database, then we can just skip it + if ok, insertErr := sqlerr.Error(err); ok { + if insertErr == sqlerr.ErrDupeKey { + continue + } + } + + if err != nil { + return err + } + + response, err := http.Get(set.IconSvgUri) + defer response.Body.Close() + if err != nil { + return nil + } + + iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION + iconFile, err := os.Create(iconFilename) + if err != nil { + return err + } + + io.Copy(iconFile, response.Body) + } + + scryfallLog("Sets updated.") + return nil +} + +func cacheBulkCardsFile(db *sql.DB, bulkData scryfall.BulkData) error { + bulkCardsResponse, err := http.Get(bulkData.DownloadUri) + defer bulkCardsResponse.Body.Close() + if err != nil { + return err + } + + cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME) + defer cacheFile.Close() + if err != nil { + return err + } + + io.Copy(cacheFile, bulkCardsResponse.Body) + + err = database.InsertOrUpdateCacheTimestampByType(db, database.CacheTypeAllCardsBulkData, bulkData.UpdatedAtTime) + if err != nil { + return err + } + + return nil +} + +func isPaper(card scryfall.Card) bool { + var paper bool = false + + for _, game := range card.Games { + if game == GAME_PAPER { + paper = true + } + } + + return paper +} + +func getScryfallCard(card scryfall.Card) []database.CardPrinting { + return database.CardPrinting{ + Id: card.Id, + Name: card.Name, + ScryfallSetCode: card.Set, + HasFoilPrinting: card.Foil, + IsPromo: card.Promo, + CollectorNumber: card.CollectorNumber, + ImageUrl: card.ImageUris["png"], + Language: card.Language, + } +} + +func updateCards(db *sql.DB, bulkData scryfall.BulkData) error { + scryfallLog("Caching bulk cards file") + err := cacheBulkCardsFile(db, bulkData) + if err != nil { + return err + } + scryfallLog("Cached bulk cards file") + + scryfallLog("Reading cached file") + cardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME) + if err != nil { + return err + } + scryfallLog("Read cached file") + + scryfallLog("Unmarshaling JSON") + var cards []scryfall.Card + err = json.Unmarshal(cardsBytes, &cards) + if err != nil { + return err + } + scryfallLog("Unmarshaled JSON") + + scryfallLog("Inserting cards") + for _, card := range cards { + // We're only interested in paper cards, so skip cards or printings of a card which + // aren't available in paper + if !isPaper(card) { + continue + } + + scryfallCard := getScryfallCard(card) + err := database.InsertCardPrinting(db, scryfallCard) + // If we already have this card in the database, then we can just skip it + if ok, insertErr := sqlerr.Error(err); ok { + if insertErr == sqlerr.ErrDupeKey { + continue + } + } + + if err != nil { + return err + } + } + scryfallLog("Inserted cards") + + return nil +} + func UpdateScryfallData(db *sql.DB) { - fmt.Println("Checking for updates...") bulkData, err := scryfall.GetBulkDataByType(scryfall.BulkDataTypeAllCards) logic.Check(err) - needsUpdate, err := logic.CheckForUpdates(db, bulkData) + needsUpdate, err := checkScryfallUpdates(db, bulkData) logic.Check(err) - if !needsUpdate { - fmt.Println("No update required.") + if needsUpdate { + scryfallLog("No update required.") return } - logic.CreateCacheDirectories() + createCacheDirectories() - err = logic.UpdateSets(db) + err = updateSets(db) logic.Check(err) - err = logic.UpdateCards(db, bulkData) + err = updateCards(db, bulkData) logic.Check(err) - fmt.Println("Update finished.") + scryfallLog("Update finished.") }