package update import ( "database/sql" "encoding/json" "io" "io/ioutil" "log" "net/http" "os" "sevenkeys/database" "sevenkeys/logic" "sevenkeys/logic/scryfall" "time" sqlerr "github.com/go-mysql/errors" ) func scryfallLog(msg string) { log.Printf("scryfall: %s\n", msg) } const CACHE_DIR string = "cache" const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/" const SET_ICON_FILE_EXTENSION string = ".svg" const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json" const GAME_PAPER = "paper" func createCacheDirectories() error { err := os.Mkdir(CACHE_DIR, os.ModePerm) if err != nil && !os.IsExist(err) { return err } err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm) if err != nil && !os.IsExist(err) { return err } return nil } func checkScryfallUpdates(db *sql.DB, bulkData scryfall.BulkData) (bool, error) { cachedFileTimestampStr, err := database.GetCacheTimestampByType(db, database.CacheTypeAllCardsBulkData) if err == sql.ErrNoRows { return true, nil } else if err != nil { return false, err } cachedFileTimestamp, err := time.Parse("2006-01-02 15:04:05", cachedFileTimestampStr) return bulkData.UpdatedAtTime.After(cachedFileTimestamp), nil } func updateSets(db *sql.DB) error { sets, err := scryfall.GetAllSets() if err != nil { return err } for _, set := range sets { // We're only interested in paper cards, so skip importing // any sets that were only released in a video game if set.Digital { continue } err := database.InsertSet(db, set) // If we already have this set in the database, then we can just skip it if ok, insertErr := sqlerr.Error(err); ok { if insertErr == sqlerr.ErrDupeKey { continue } } if err != nil { return err } response, err := http.Get(set.IconSvgUri) defer response.Body.Close() if err != nil { return nil } iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION iconFile, err := os.Create(iconFilename) if err != nil { return err } io.Copy(iconFile, response.Body) } scryfallLog("Sets updated.") return nil } func cacheBulkCardsFile(db *sql.DB, bulkData scryfall.BulkData) error { bulkCardsResponse, err := http.Get(bulkData.DownloadUri) defer bulkCardsResponse.Body.Close() if err != nil { return err } cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME) defer cacheFile.Close() if err != nil { return err } io.Copy(cacheFile, bulkCardsResponse.Body) err = database.InsertOrUpdateCacheTimestampByType(db, database.CacheTypeAllCardsBulkData, bulkData.UpdatedAtTime) if err != nil { return err } return nil } func isPaper(card scryfall.Card) bool { var paper bool = false for _, game := range card.Games { if game == GAME_PAPER { paper = true } } return paper } func getScryfallCard(card scryfall.Card) database.ScryfallCard { return database.ScryfallCard{ Id: card.Id, Name: card.Name, ScryfallSetCode: card.Set, HasFoilPrinting: card.Foil, IsPromo: card.Promo, CollectorNumber: card.CollectorNumber, ImageUrl: card.ImageUris["png"], Language: card.Language, } } func updateCards(db *sql.DB, bulkData scryfall.BulkData) error { scryfallLog("Caching bulk cards file") err := cacheBulkCardsFile(db, bulkData) if err != nil { return err } scryfallLog("Cached bulk cards file") scryfallLog("Reading cached file") cardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME) if err != nil { return err } scryfallLog("Read cached file") scryfallLog("Unmarshaling JSON") var cards []scryfall.Card err = json.Unmarshal(cardsBytes, &cards) if err != nil { return err } scryfallLog("Unmarshaled JSON") scryfallLog("Inserting cards") for _, card := range cards { // We're only interested in paper cards, so skip cards or printings of a card which // aren't available in paper if !isPaper(card) { continue } scryfallCard := getScryfallCard(card) err := database.InsertScryfallCard(db, scryfallCard) // If we already have this card in the database, then we can just skip it if ok, insertErr := sqlerr.Error(err); ok { if insertErr == sqlerr.ErrDupeKey { continue } } if err != nil { return err } } scryfallLog("Inserted cards") return nil } func UpdateScryfallData(db *sql.DB) { bulkData, err := scryfall.GetBulkDataByType(scryfall.BulkDataTypeAllCards) logic.Check(err) needsUpdate, err := checkScryfallUpdates(db, bulkData) logic.Check(err) if needsUpdate { scryfallLog("No update required.") return } createCacheDirectories() err = updateSets(db) logic.Check(err) err = updateCards(db, bulkData) logic.Check(err) scryfallLog("Update finished.") }