TheMathemagicians/sevenkeys/cmd/importer/main.go

204 lines
4.7 KiB
Go
Raw Normal View History

2024-04-04 13:55:12 +00:00
package main
import (
2024-05-21 11:33:14 +00:00
"database/sql"
2024-05-21 14:15:58 +00:00
"encoding/json"
2024-05-21 11:33:14 +00:00
"io"
2024-05-21 14:15:58 +00:00
"io/ioutil"
2024-05-21 11:17:50 +00:00
"log"
2024-05-21 11:33:14 +00:00
"net/http"
2024-05-21 11:28:08 +00:00
"os"
2024-05-21 11:24:16 +00:00
"sevenkeys/database"
2024-05-21 13:31:54 +00:00
"sevenkeys/database/entities"
2024-05-21 11:33:14 +00:00
"sevenkeys/database/operations"
2024-05-21 11:24:16 +00:00
"sevenkeys/scryfall/methods"
2024-05-21 11:33:14 +00:00
"sevenkeys/scryfall/types"
2024-05-21 13:31:54 +00:00
"time"
2024-04-04 13:55:12 +00:00
)
2024-05-21 11:19:01 +00:00
const CACHE_DIR string = "cache"
const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/"
const SET_ICON_FILE_EXTENSION string = ".svg"
const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json"
const GAME_PAPER = "paper"
2024-05-21 11:17:50 +00:00
func check(err error) {
if err != nil {
log.Fatal(err)
}
}
2024-05-21 11:28:08 +00:00
func createCacheDirectories() error {
err := os.Mkdir(CACHE_DIR, os.ModePerm)
if err != nil && !os.IsExist(err) {
return err
}
err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm)
if err != nil && !os.IsExist(err) {
return err
}
return nil
}
2024-05-21 11:33:14 +00:00
func importSets(db *sql.DB, sets []types.Set) error {
for _, set := range sets {
// We're only interested in paper cards, so skip importing
// any sets that were only released in a video game
if set.Digital {
continue
}
log.Println("Importing " + set.Code)
2024-05-21 12:25:08 +00:00
err := operations.InsertOrUpdateSet(db, set)
2024-05-21 11:33:14 +00:00
if err != nil {
return err
}
log.Println("Downloading logo for " + set.Code)
response, err := http.Get(set.IconSvgUri)
defer response.Body.Close()
if err != nil {
return err
}
iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION
iconFile, err := os.Create(iconFilename)
if err != nil {
return err
}
io.Copy(iconFile, response.Body)
log.Println("Finished importing " + set.Code)
}
2024-05-21 12:25:08 +00:00
return nil
2024-05-21 11:33:14 +00:00
}
2024-05-21 13:31:54 +00:00
func cardsUpdatedSinceCache(db *sql.DB, updatedAtTimestamp time.Time) (bool, error) {
cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData)
if err != nil {
return false, err
}
return updatedAtTimestamp.After(cachedFileTimestamp), nil
}
func cacheAllCardsFile(db *sql.DB, uri string, updatedAtTimestamp time.Time) error {
log.Printf("Downloading bulk card data...")
bulkCardsResponse, err := http.Get(uri)
if err != nil {
return err
}
log.Printf("Downloaded bulk card data.")
log.Printf("Writing card data to cache file...")
cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
if err != nil {
return err
}
defer bulkCardsResponse.Body.Close()
defer cacheFile.Close()
io.Copy(cacheFile, bulkCardsResponse.Body)
log.Printf("Cache file written.")
log.Printf("Recording timestamp...")
err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
if err != nil {
return err
}
log.Printf("Timestamp recorded.")
return nil
}
2024-05-21 14:27:05 +00:00
func importCards(db *sql.DB, cards []types.Card) error {
for _, card := range cards {
// We're only interested in paper cards, so skip cards or printings of a card which
// aren't available in paper
if !isPaper(card) {
continue
}
2024-05-21 14:27:05 +00:00
err := operations.InsertCard(db, card)
if err != nil {
return err
}
}
return nil
}
func isPaper(card types.Card) bool {
var paper bool = false
for _, game := range card.Games {
if game == GAME_PAPER {
paper = true
}
}
return paper
}
2024-04-04 13:55:12 +00:00
func main() {
2024-05-21 11:24:16 +00:00
log.Println("Connecting to database...")
db := database.GetDatabaseFromConfig("config.json")
log.Println("Connected.")
2024-05-21 11:28:08 +00:00
log.Println("Creating cache directories...")
2024-05-21 12:25:08 +00:00
err := createCacheDirectories()
2024-05-21 11:28:08 +00:00
check(err)
log.Println("Created cache directories.")
2024-05-21 11:24:16 +00:00
2024-05-21 14:27:36 +00:00
log.Println("Downloading set data from Scryfall...")
sets, err := methods.GetSets()
check(err)
log.Println("Downloaded set data.")
2024-05-21 13:31:54 +00:00
2024-05-21 14:27:36 +00:00
log.Println("Importing set data...")
err = importSets(db, sets)
check(err)
log.Println("Imported sets.")
2024-05-21 13:31:54 +00:00
log.Println("Downloading bulk data entry...")
allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards)
2024-05-21 11:33:14 +00:00
check(err)
2024-05-21 13:31:54 +00:00
log.Println("Downloaded bulk data entry.")
2024-05-21 11:33:14 +00:00
2024-05-21 13:31:54 +00:00
updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
2024-05-21 11:33:14 +00:00
check(err)
2024-05-21 13:31:54 +00:00
updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second) // Remove the fractional seconds component from the timestamp
2024-05-21 11:24:16 +00:00
2024-05-21 13:31:54 +00:00
log.Println("Checking for redownload...")
updated, err := cardsUpdatedSinceCache(db, updatedAtTimestamp)
check(err)
if updated {
log.Printf("Bulk data has been updated since last cache, redownloading.")
err = cacheAllCardsFile(db, allCardsBulkData.DownloadUri, updatedAtTimestamp)
check(err)
} else {
log.Printf("Bulk data has not been updated. Skipping download.")
}
2024-05-21 11:24:16 +00:00
2024-05-21 14:15:58 +00:00
log.Printf("Unmarsaling file into slice...")
allCardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME)
check(err)
var allCards []types.Card
err = json.Unmarshal(allCardsBytes, &allCards)
check(err)
log.Printf("Unmarshaled file.")
log.Printf("Importing card data into database...")
2024-05-21 14:27:05 +00:00
err = importCards(db, allCards)
2024-05-21 14:15:58 +00:00
check(err)
log.Printf("Imported card data.")
2024-04-04 13:55:12 +00:00
}