TheMathemagicians/sevenkeys/update/scryfall.go

package update

import (
	"database/sql"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sevenkeys/database"
	"sevenkeys/logic"
	"sevenkeys/logic/scryfall"
	"time"

	sqlerr "github.com/go-mysql/errors"
)

func scryfallLog(msg string) {
	log.Printf("scryfall: %s\n", msg)
}

const CACHE_DIR string = "cache"
const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/"
const SET_ICON_FILE_EXTENSION string = ".svg"
const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json"
const GAME_PAPER = "paper"

func createCacheDirectories() error {
	err := os.Mkdir(CACHE_DIR, os.ModePerm)
	if err != nil && !os.IsExist(err) {
		return err
	}

	err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm)
	if err != nil && !os.IsExist(err) {
		return err
	}

	return nil
}

func checkScryfallUpdates(db *sql.DB, bulkData scryfall.BulkData) (bool, error) {
	cachedFileTimestampStr, err := database.GetCacheTimestampByType(db, database.CacheTypeAllCardsBulkData)
	if err == sql.ErrNoRows {
		return true, nil
	} else if err != nil {
		return false, err
	}

	cachedFileTimestamp, err := time.Parse("2006-01-02 15:04:05", cachedFileTimestampStr)

	return bulkData.UpdatedAtTime.After(cachedFileTimestamp), nil
}

func updateSets(db *sql.DB) error {
	sets, err := scryfall.GetAllSets()
	if err != nil {
		return err
	}

	for _, set := range sets {
		// We're only interested in paper cards, so skip importing
		// any sets that were only released in a video game
		if set.Digital {
			continue
		}

		err := database.InsertSet(db, set)
		// If we already have this set in the database, then we can just skip it
		if ok, insertErr := sqlerr.Error(err); ok {
			if insertErr == sqlerr.ErrDupeKey {
				continue
			}
		}

		if err != nil {
			return err
		}

		response, err := http.Get(set.IconSvgUri)
		defer response.Body.Close()
		if err != nil {
			return nil
		}

		iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION
		iconFile, err := os.Create(iconFilename)
		if err != nil {
			return err
		}

		io.Copy(iconFile, response.Body)
	}

	scryfallLog("Sets updated.")
	return nil
}

func cacheBulkCardsFile(db *sql.DB, bulkData scryfall.BulkData) error {
	bulkCardsResponse, err := http.Get(bulkData.DownloadUri)
	defer bulkCardsResponse.Body.Close()
	if err != nil {
		return err
	}

	cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
	defer cacheFile.Close()
	if err != nil {
		return err
	}

	io.Copy(cacheFile, bulkCardsResponse.Body)

	err = database.InsertOrUpdateCacheTimestampByType(db, database.CacheTypeAllCardsBulkData, bulkData.UpdatedAtTime)
	if err != nil {
		return err
	}

	return nil
}

func isPaper(card scryfall.Card) bool {
	var paper bool = false

	for _, game := range card.Games {
		if game == GAME_PAPER {
			paper = true
		}
	}

	return paper
}

func getScryfallCard(card scryfall.Card) database.ScryfallCard {
	return database.ScryfallCard{
		Id:              card.Id,
		Name:            card.Name,
		ScryfallSetCode: card.Set,
		HasFoilPrinting: card.Foil,
		IsPromo:         card.Promo,
		CollectorNumber: card.CollectorNumber,
		ImageUrl:        card.ImageUris["png"],
		Language:        card.Language,
	}
}

func updateCards(db *sql.DB, bulkData scryfall.BulkData) error {
	scryfallLog("Caching bulk cards file")
	err := cacheBulkCardsFile(db, bulkData)
	if err != nil {
		return err
	}
	scryfallLog("Cached bulk cards file")

	scryfallLog("Reading cached file")
	cardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME)
	if err != nil {
		return err
	}
	scryfallLog("Read cached file")

	scryfallLog("Unmarshaling JSON")
	var cards []scryfall.Card
	err = json.Unmarshal(cardsBytes, &cards)
	if err != nil {
		return err
	}
	scryfallLog("Unmarshaled JSON")

	scryfallLog("Inserting cards")
	for _, card := range cards {
		// We're only interested in paper cards, so skip cards or printings of a card which
		// aren't available in paper
		if !isPaper(card) {
			continue
		}

		scryfallCard := getScryfallCard(card)
		err := database.InsertScryfallCard(db, scryfallCard)
		// If we already have this card in the database, then we can just skip it
		if ok, insertErr := sqlerr.Error(err); ok {
			if insertErr == sqlerr.ErrDupeKey {
				continue
			}
		}

		if err != nil {
			return err
		}
	}
	scryfallLog("Inserted cards")

	return nil
}

func UpdateScryfallData(db *sql.DB) {
	bulkData, err := scryfall.GetBulkDataByType(scryfall.BulkDataTypeAllCards)
	logic.Check(err)

	needsUpdate, err := checkScryfallUpdates(db, bulkData)
	logic.Check(err)

	if !needsUpdate {
		scryfallLog("No update required.")
		return
	}

	createCacheDirectories()

	err = updateSets(db)
	logic.Check(err)

	err = updateCards(db, bulkData)
	logic.Check(err)

	scryfallLog("Update finished.")
}