diff --git a/sevenkeys/cmd/importcards/main.go b/sevenkeys/cmd/importcards/main.go index bc90436..095d44b 100644 --- a/sevenkeys/cmd/importcards/main.go +++ b/sevenkeys/cmd/importcards/main.go @@ -2,17 +2,10 @@ package main import ( "encoding/json" - "io" "io/ioutil" "log" - "net/http" - "os" - "sevenkeys/database" - "sevenkeys/database/entities" "sevenkeys/database/operations" - "sevenkeys/scryfall/methods" "sevenkeys/scryfall/types" - "time" ) func check(err error) { @@ -22,49 +15,7 @@ func check(err error) { } func main() { - log.Printf("Getting bulk data listing...") - allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards) - check(err) - log.Printf("Got bulk data listing.") - - log.Printf("Checking if we need to download bulk data...") - updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt) - check(err) - // Remove the fractional seconds component from the downloaded timestamp because it isn't saved in the cache table - updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second) - - db := database.GetDatabaseFromConfig("config.json") - - cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData) - check(err) - if updatedAtTimestamp.After(cachedFileTimestamp) { - log.Printf("Bulk data has been updated since last cache, redownloading.") - - err = os.RemoveAll(CACHE_DIR) - check(err) - - err := os.Mkdir(CACHE_DIR, os.ModePerm) - check(err) - - log.Printf("Downloading bulk card data...") - bulkCardsResponse, err := http.Get(allCardsBulkData.DownloadUri) - check(err) - log.Printf("Downloaded bulk card data.") - - log.Printf("Writing card data to cache file...") - cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME) - check(err) - - defer bulkCardsResponse.Body.Close() - defer cacheFile.Close() - io.Copy(cacheFile, bulkCardsResponse.Body) - log.Printf("Cache file written.") - - log.Printf("Recording timestamp...") - err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp) - check(err) - log.Printf("Timestamp recorded.") } else { log.Printf("Bulk data has not been updated. Skipping download.") } diff --git a/sevenkeys/main.go b/sevenkeys/main.go index d1c90bb..886d859 100644 --- a/sevenkeys/main.go +++ b/sevenkeys/main.go @@ -7,9 +7,11 @@ import ( "net/http" "os" "sevenkeys/database" + "sevenkeys/database/entities" "sevenkeys/database/operations" "sevenkeys/scryfall/methods" "sevenkeys/scryfall/types" + "time" ) const CACHE_DIR string = "cache" @@ -73,6 +75,44 @@ func importSets(db *sql.DB, sets []types.Set) error { return nil } +func cardsUpdatedSinceCache(db *sql.DB, updatedAtTimestamp time.Time) (bool, error) { + cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData) + if err != nil { + return false, err + } + + return updatedAtTimestamp.After(cachedFileTimestamp), nil +} + +func cacheAllCardsFile(db *sql.DB, uri string, updatedAtTimestamp time.Time) error { + log.Printf("Downloading bulk card data...") + bulkCardsResponse, err := http.Get(uri) + if err != nil { + return err + } + log.Printf("Downloaded bulk card data.") + + log.Printf("Writing card data to cache file...") + cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME) + if err != nil { + return err + } + + defer bulkCardsResponse.Body.Close() + defer cacheFile.Close() + io.Copy(cacheFile, bulkCardsResponse.Body) + log.Printf("Cache file written.") + + log.Printf("Recording timestamp...") + err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp) + if err != nil { + return err + } + log.Printf("Timestamp recorded.") + + return nil +} + func main() { log.Println("Connecting to database...") db := database.GetDatabaseFromConfig("config.json") @@ -83,18 +123,39 @@ func main() { check(err) log.Println("Created cache directories.") - log.Println("Downloading set data from Scryfall...") - sets, err := methods.GetSets() - check(err) - log.Println("Downloaded set data.") + /* + log.Println("Downloading set data from Scryfall...") + sets, err := methods.GetSets() + check(err) + log.Println("Downloaded set data.") - log.Println("Importing set data...") - err = importSets(db, sets) - check(err) - log.Println("Imported sets.") + log.Println("Importing set data...") + err = importSets(db, sets) + check(err) + log.Println("Imported sets.") + */ - // Check whether card data has been updated since last download - // If yes, redownload and recache the bulk data + log.Println("Downloading bulk data entry...") + allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards) + check(err) + log.Println("Downloaded bulk data entry.") + + updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt) + check(err) + updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second) // Remove the fractional seconds component from the timestamp + + log.Println("Checking for redownload...") + updated, err := cardsUpdatedSinceCache(db, updatedAtTimestamp) + check(err) + + if updated { + log.Printf("Bulk data has been updated since last cache, redownloading.") + return + err = cacheAllCardsFile(db, allCardsBulkData.DownloadUri, updatedAtTimestamp) + check(err) + } else { + log.Printf("Bulk data has not been updated. Skipping download.") + } // Unmarshal cached file into Golang struct // Import cards into database