Cache bulk data file

This commit is contained in:
The Magician 2024-05-21 14:31:54 +01:00
parent 01e06c0152
commit d1fa838bfe
2 changed files with 71 additions and 59 deletions

View File

@ -2,17 +2,10 @@ package main
import ( import (
"encoding/json" "encoding/json"
"io"
"io/ioutil" "io/ioutil"
"log" "log"
"net/http"
"os"
"sevenkeys/database"
"sevenkeys/database/entities"
"sevenkeys/database/operations" "sevenkeys/database/operations"
"sevenkeys/scryfall/methods"
"sevenkeys/scryfall/types" "sevenkeys/scryfall/types"
"time"
) )
func check(err error) { func check(err error) {
@ -22,49 +15,7 @@ func check(err error) {
} }
func main() { func main() {
log.Printf("Getting bulk data listing...")
allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards)
check(err)
log.Printf("Got bulk data listing.")
log.Printf("Checking if we need to download bulk data...")
updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
check(err)
// Remove the fractional seconds component from the downloaded timestamp because it isn't saved in the cache table
updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second)
db := database.GetDatabaseFromConfig("config.json")
cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData)
check(err)
if updatedAtTimestamp.After(cachedFileTimestamp) { if updatedAtTimestamp.After(cachedFileTimestamp) {
log.Printf("Bulk data has been updated since last cache, redownloading.")
err = os.RemoveAll(CACHE_DIR)
check(err)
err := os.Mkdir(CACHE_DIR, os.ModePerm)
check(err)
log.Printf("Downloading bulk card data...")
bulkCardsResponse, err := http.Get(allCardsBulkData.DownloadUri)
check(err)
log.Printf("Downloaded bulk card data.")
log.Printf("Writing card data to cache file...")
cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
check(err)
defer bulkCardsResponse.Body.Close()
defer cacheFile.Close()
io.Copy(cacheFile, bulkCardsResponse.Body)
log.Printf("Cache file written.")
log.Printf("Recording timestamp...")
err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
check(err)
log.Printf("Timestamp recorded.")
} else { } else {
log.Printf("Bulk data has not been updated. Skipping download.") log.Printf("Bulk data has not been updated. Skipping download.")
} }

View File

@ -7,9 +7,11 @@ import (
"net/http" "net/http"
"os" "os"
"sevenkeys/database" "sevenkeys/database"
"sevenkeys/database/entities"
"sevenkeys/database/operations" "sevenkeys/database/operations"
"sevenkeys/scryfall/methods" "sevenkeys/scryfall/methods"
"sevenkeys/scryfall/types" "sevenkeys/scryfall/types"
"time"
) )
const CACHE_DIR string = "cache" const CACHE_DIR string = "cache"
@ -73,6 +75,44 @@ func importSets(db *sql.DB, sets []types.Set) error {
return nil return nil
} }
func cardsUpdatedSinceCache(db *sql.DB, updatedAtTimestamp time.Time) (bool, error) {
cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData)
if err != nil {
return false, err
}
return updatedAtTimestamp.After(cachedFileTimestamp), nil
}
func cacheAllCardsFile(db *sql.DB, uri string, updatedAtTimestamp time.Time) error {
log.Printf("Downloading bulk card data...")
bulkCardsResponse, err := http.Get(uri)
if err != nil {
return err
}
log.Printf("Downloaded bulk card data.")
log.Printf("Writing card data to cache file...")
cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
if err != nil {
return err
}
defer bulkCardsResponse.Body.Close()
defer cacheFile.Close()
io.Copy(cacheFile, bulkCardsResponse.Body)
log.Printf("Cache file written.")
log.Printf("Recording timestamp...")
err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
if err != nil {
return err
}
log.Printf("Timestamp recorded.")
return nil
}
func main() { func main() {
log.Println("Connecting to database...") log.Println("Connecting to database...")
db := database.GetDatabaseFromConfig("config.json") db := database.GetDatabaseFromConfig("config.json")
@ -83,18 +123,39 @@ func main() {
check(err) check(err)
log.Println("Created cache directories.") log.Println("Created cache directories.")
log.Println("Downloading set data from Scryfall...") /*
sets, err := methods.GetSets() log.Println("Downloading set data from Scryfall...")
check(err) sets, err := methods.GetSets()
log.Println("Downloaded set data.") check(err)
log.Println("Downloaded set data.")
log.Println("Importing set data...") log.Println("Importing set data...")
err = importSets(db, sets) err = importSets(db, sets)
check(err) check(err)
log.Println("Imported sets.") log.Println("Imported sets.")
*/
// Check whether card data has been updated since last download log.Println("Downloading bulk data entry...")
// If yes, redownload and recache the bulk data allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards)
check(err)
log.Println("Downloaded bulk data entry.")
updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
check(err)
updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second) // Remove the fractional seconds component from the timestamp
log.Println("Checking for redownload...")
updated, err := cardsUpdatedSinceCache(db, updatedAtTimestamp)
check(err)
if updated {
log.Printf("Bulk data has been updated since last cache, redownloading.")
return
err = cacheAllCardsFile(db, allCardsBulkData.DownloadUri, updatedAtTimestamp)
check(err)
} else {
log.Printf("Bulk data has not been updated. Skipping download.")
}
// Unmarshal cached file into Golang struct // Unmarshal cached file into Golang struct
// Import cards into database // Import cards into database