Move update logic to update package
This commit is contained in:
parent
6814b23018
commit
ec6e6b7f6a
|
|
@ -1,22 +0,0 @@
|
|||
package logic
|
||||
|
||||
import "os"
|
||||
|
||||
const CACHE_DIR string = "cache"
|
||||
const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/"
|
||||
const SET_ICON_FILE_EXTENSION string = ".svg"
|
||||
const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json"
|
||||
|
||||
func CreateCacheDirectories() error {
|
||||
err := os.Mkdir(CACHE_DIR, os.ModePerm)
|
||||
if err != nil && !os.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm)
|
||||
if err != nil && !os.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1,197 +0,0 @@
|
|||
package logic
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"sevenkeys/database"
|
||||
"sevenkeys/logic/scryfall"
|
||||
"time"
|
||||
|
||||
sqlerr "github.com/go-mysql/errors"
|
||||
)
|
||||
|
||||
const GAME_PAPER = "paper"
|
||||
|
||||
func CheckForUpdates(db *sql.DB, bulkData scryfall.BulkData) (bool, error) {
|
||||
// TODO: We also want to update if:
|
||||
// - there is no cached version of the all-cards.json file or it is empty
|
||||
// - The set icon files are missing
|
||||
cachedFileTimestampStr, err := database.GetCacheTimestampByType(db, database.CacheTypeAllCardsBulkData)
|
||||
if err == sql.ErrNoRows {
|
||||
return true, nil
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
cachedFileTimestamp, err := time.Parse("2006-01-02 15:04:05", cachedFileTimestampStr)
|
||||
|
||||
return bulkData.UpdatedAtTime.After(cachedFileTimestamp), nil
|
||||
}
|
||||
|
||||
func UpdateSets(db *sql.DB) error {
|
||||
sets, err := scryfall.GetAllSets()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, set := range sets {
|
||||
// We're only interested in paper cards, so skip importing
|
||||
// any sets that were only released in a video game
|
||||
if set.Digital {
|
||||
continue
|
||||
}
|
||||
|
||||
err := database.InsertSet(db, set)
|
||||
// If we already have this set in the database, then we can just skip it
|
||||
if ok, insertErr := sqlerr.Error(err); ok {
|
||||
if insertErr == sqlerr.ErrDupeKey {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
response, err := http.Get(set.IconSvgUri)
|
||||
defer response.Body.Close()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION
|
||||
iconFile, err := os.Create(iconFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
io.Copy(iconFile, response.Body)
|
||||
}
|
||||
|
||||
fmt.Println("Sets updated.")
|
||||
return nil
|
||||
}
|
||||
|
||||
func cacheBulkCardsFile(db *sql.DB, bulkData scryfall.BulkData) error {
|
||||
bulkCardsResponse, err := http.Get(bulkData.DownloadUri)
|
||||
defer bulkCardsResponse.Body.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
|
||||
defer cacheFile.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
io.Copy(cacheFile, bulkCardsResponse.Body)
|
||||
|
||||
err = database.InsertOrUpdateCacheTimestampByType(db, database.CacheTypeAllCardsBulkData, bulkData.UpdatedAtTime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isPaper(card scryfall.Card) bool {
|
||||
var paper bool = false
|
||||
|
||||
for _, game := range card.Games {
|
||||
if game == GAME_PAPER {
|
||||
paper = true
|
||||
}
|
||||
}
|
||||
|
||||
return paper
|
||||
}
|
||||
|
||||
func getCardPrintings(card scryfall.Card) []database.CardPrinting {
|
||||
var printings []database.CardPrinting
|
||||
|
||||
if card.Foil {
|
||||
printings = append(printings, database.CardPrinting{
|
||||
Id: card.Id + "f",
|
||||
Name: card.Name,
|
||||
SetCode: card.Set,
|
||||
IsFoil: true,
|
||||
IsPromo: card.Promo,
|
||||
CollectorNumber: card.CollectorNumber,
|
||||
ImageUrl: card.ImageUris["png"],
|
||||
Language: card.Language,
|
||||
})
|
||||
}
|
||||
|
||||
if card.NonFoil {
|
||||
printings = append(printings, database.CardPrinting{
|
||||
Id: card.Id + "n",
|
||||
Name: card.Name,
|
||||
SetCode: card.Set,
|
||||
IsFoil: false,
|
||||
IsPromo: card.Promo,
|
||||
CollectorNumber: card.CollectorNumber,
|
||||
Language: card.Language,
|
||||
})
|
||||
}
|
||||
|
||||
return printings
|
||||
}
|
||||
|
||||
func UpdateCards(db *sql.DB, bulkData scryfall.BulkData) error {
|
||||
log.Println("Caching bulk cards file")
|
||||
err := cacheBulkCardsFile(db, bulkData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Println("Cached bulk cards file")
|
||||
|
||||
log.Println("Reading cached file")
|
||||
cardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Println("Read cached file")
|
||||
|
||||
log.Println("Unmarshaling JSON")
|
||||
var cards []scryfall.Card
|
||||
err = json.Unmarshal(cardsBytes, &cards)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Println("Unmarshaled JSON")
|
||||
|
||||
log.Println("INSERTing cards")
|
||||
for _, card := range cards {
|
||||
// We're only interested in paper cards, so skip cards or printings of a card which
|
||||
// aren't available in paper
|
||||
if !isPaper(card) {
|
||||
continue
|
||||
}
|
||||
|
||||
cardPrintings := getCardPrintings(card)
|
||||
|
||||
for _, cardPrinting := range cardPrintings {
|
||||
err := database.InsertCardPrinting(db, cardPrinting)
|
||||
// If we already have this card in the database, then we can just skip it
|
||||
if ok, insertErr := sqlerr.Error(err); ok {
|
||||
if insertErr == sqlerr.ErrDupeKey {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
log.Println("INSERTed cards")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -2,31 +2,215 @@ package update
|
|||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"sevenkeys/database"
|
||||
"sevenkeys/logic"
|
||||
"sevenkeys/logic/scryfall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func scryfallLog(msg string) {
|
||||
log.Printf("scryfall: %s\n", msg)
|
||||
}
|
||||
|
||||
const CACHE_DIR string = "cache"
|
||||
const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/"
|
||||
const SET_ICON_FILE_EXTENSION string = ".svg"
|
||||
const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json"
|
||||
const GAME_PAPER = "paper"
|
||||
|
||||
func createCacheDirectories() error {
|
||||
err := os.Mkdir(CACHE_DIR, os.ModePerm)
|
||||
if err != nil && !os.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm)
|
||||
if err != nil && !os.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkScryfallUpdates(db *sql.DB, bulkData scryfall.BulkData) (bool, error) {
|
||||
cachedFileTimestampStr, err := database.GetCacheTimestampByType(db, database.CacheTypeAllCardsBulkData)
|
||||
if err == sql.ErrNoRows {
|
||||
return true, nil
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
cachedFileTimestamp, err := time.Parse("2006-01-02 15:04:05", cachedFileTimestampStr)
|
||||
|
||||
return bulkData.UpdatedAtTime.After(cachedFileTimestamp), nil
|
||||
}
|
||||
|
||||
func updateSets(db *sql.DB) error {
|
||||
sets, err := scryfall.GetAllSets()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, set := range sets {
|
||||
// We're only interested in paper cards, so skip importing
|
||||
// any sets that were only released in a video game
|
||||
if set.Digital {
|
||||
continue
|
||||
}
|
||||
|
||||
err := database.InsertSet(db, set)
|
||||
// If we already have this set in the database, then we can just skip it
|
||||
if ok, insertErr := sqlerr.Error(err); ok {
|
||||
if insertErr == sqlerr.ErrDupeKey {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
response, err := http.Get(set.IconSvgUri)
|
||||
defer response.Body.Close()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION
|
||||
iconFile, err := os.Create(iconFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
io.Copy(iconFile, response.Body)
|
||||
}
|
||||
|
||||
scryfallLog("Sets updated.")
|
||||
return nil
|
||||
}
|
||||
|
||||
func cacheBulkCardsFile(db *sql.DB, bulkData scryfall.BulkData) error {
|
||||
bulkCardsResponse, err := http.Get(bulkData.DownloadUri)
|
||||
defer bulkCardsResponse.Body.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
|
||||
defer cacheFile.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
io.Copy(cacheFile, bulkCardsResponse.Body)
|
||||
|
||||
err = database.InsertOrUpdateCacheTimestampByType(db, database.CacheTypeAllCardsBulkData, bulkData.UpdatedAtTime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isPaper(card scryfall.Card) bool {
|
||||
var paper bool = false
|
||||
|
||||
for _, game := range card.Games {
|
||||
if game == GAME_PAPER {
|
||||
paper = true
|
||||
}
|
||||
}
|
||||
|
||||
return paper
|
||||
}
|
||||
|
||||
func getScryfallCard(card scryfall.Card) []database.CardPrinting {
|
||||
return database.CardPrinting{
|
||||
Id: card.Id,
|
||||
Name: card.Name,
|
||||
ScryfallSetCode: card.Set,
|
||||
HasFoilPrinting: card.Foil,
|
||||
IsPromo: card.Promo,
|
||||
CollectorNumber: card.CollectorNumber,
|
||||
ImageUrl: card.ImageUris["png"],
|
||||
Language: card.Language,
|
||||
}
|
||||
}
|
||||
|
||||
func updateCards(db *sql.DB, bulkData scryfall.BulkData) error {
|
||||
scryfallLog("Caching bulk cards file")
|
||||
err := cacheBulkCardsFile(db, bulkData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
scryfallLog("Cached bulk cards file")
|
||||
|
||||
scryfallLog("Reading cached file")
|
||||
cardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
scryfallLog("Read cached file")
|
||||
|
||||
scryfallLog("Unmarshaling JSON")
|
||||
var cards []scryfall.Card
|
||||
err = json.Unmarshal(cardsBytes, &cards)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
scryfallLog("Unmarshaled JSON")
|
||||
|
||||
scryfallLog("Inserting cards")
|
||||
for _, card := range cards {
|
||||
// We're only interested in paper cards, so skip cards or printings of a card which
|
||||
// aren't available in paper
|
||||
if !isPaper(card) {
|
||||
continue
|
||||
}
|
||||
|
||||
scryfallCard := getScryfallCard(card)
|
||||
err := database.InsertCardPrinting(db, scryfallCard)
|
||||
// If we already have this card in the database, then we can just skip it
|
||||
if ok, insertErr := sqlerr.Error(err); ok {
|
||||
if insertErr == sqlerr.ErrDupeKey {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
scryfallLog("Inserted cards")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func UpdateScryfallData(db *sql.DB) {
|
||||
fmt.Println("Checking for updates...")
|
||||
bulkData, err := scryfall.GetBulkDataByType(scryfall.BulkDataTypeAllCards)
|
||||
logic.Check(err)
|
||||
|
||||
needsUpdate, err := logic.CheckForUpdates(db, bulkData)
|
||||
needsUpdate, err := checkScryfallUpdates(db, bulkData)
|
||||
logic.Check(err)
|
||||
|
||||
if !needsUpdate {
|
||||
fmt.Println("No update required.")
|
||||
if needsUpdate {
|
||||
scryfallLog("No update required.")
|
||||
return
|
||||
}
|
||||
|
||||
logic.CreateCacheDirectories()
|
||||
createCacheDirectories()
|
||||
|
||||
err = logic.UpdateSets(db)
|
||||
err = updateSets(db)
|
||||
logic.Check(err)
|
||||
|
||||
err = logic.UpdateCards(db, bulkData)
|
||||
err = updateCards(db, bulkData)
|
||||
logic.Check(err)
|
||||
|
||||
fmt.Println("Update finished.")
|
||||
scryfallLog("Update finished.")
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue