TheMathemagicians/sevenkeys/cmd/importer/main.go

package main

import (
	"database/sql"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sevenkeys/database"
	"sevenkeys/database/entities"
	"sevenkeys/database/operations"
	"sevenkeys/scryfall/methods"
	"sevenkeys/scryfall/types"
	"time"
)

const CACHE_DIR string = "cache"

const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/"
const SET_ICON_FILE_EXTENSION string = ".svg"

const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json"

const GAME_PAPER = "paper"

func check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func createCacheDirectories() error {
	err := os.Mkdir(CACHE_DIR, os.ModePerm)
	if err != nil && !os.IsExist(err) {
		return err
	}

	err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm)
	if err != nil && !os.IsExist(err) {
		return err
	}

	return nil
}

func importSets(db *sql.DB, sets []types.Set) error {
	for _, set := range sets {
		// We're only interested in paper cards, so skip importing
		// any sets that were only released in a video game
		if set.Digital {
			continue
		}

		log.Println("Importing " + set.Code)
		err := operations.InsertOrUpdateSet(db, set)
		if err != nil {
			return err
		}

		log.Println("Downloading logo for " + set.Code)
		response, err := http.Get(set.IconSvgUri)
		defer response.Body.Close()
		if err != nil {
			return err
		}

		iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION
		iconFile, err := os.Create(iconFilename)
		if err != nil {
			return err
		}

		io.Copy(iconFile, response.Body)
		log.Println("Finished importing " + set.Code)
	}

	return nil
}

func cardsUpdatedSinceCache(db *sql.DB, updatedAtTimestamp time.Time) (bool, error) {
	cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData)
	if err != nil {
		return false, err
	}

	return updatedAtTimestamp.After(cachedFileTimestamp), nil
}

func cacheAllCardsFile(db *sql.DB, uri string, updatedAtTimestamp time.Time) error {
	log.Printf("Downloading bulk card data...")
	bulkCardsResponse, err := http.Get(uri)
	if err != nil {
		return err
	}
	log.Printf("Downloaded bulk card data.")

	log.Printf("Writing card data to cache file...")
	cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
	if err != nil {
		return err
	}

	defer bulkCardsResponse.Body.Close()
	defer cacheFile.Close()
	io.Copy(cacheFile, bulkCardsResponse.Body)
	log.Printf("Cache file written.")

	log.Printf("Recording timestamp...")
	err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
	if err != nil {
		return err
	}
	log.Printf("Timestamp recorded.")

	return nil
}

func importCards(db *sql.DB, cards []types.Card) error {
	for _, card := range cards {
		// We're only interested in paper cards, so skip cards or printings of a card which
		// aren't available in paper
		if !isPaper(card) {
			continue
		}

		err := operations.InsertCard(db, card)
		if err != nil {
			return err
		}
	}

	return nil
}

func isPaper(card types.Card) bool {
	var paper bool = false

	for _, game := range card.Games {
		if game == GAME_PAPER {
			paper = true
		}
	}

	return paper
}

func main() {
	log.Println("Connecting to database...")
	db := database.GetDatabaseFromConfig("config.json")
	log.Println("Connected.")

	log.Println("Creating cache directories...")
	err := createCacheDirectories()
	check(err)
	log.Println("Created cache directories.")

	log.Println("Downloading set data from Scryfall...")
	sets, err := methods.GetSets()
	check(err)
	log.Println("Downloaded set data.")

	log.Println("Importing set data...")
	err = importSets(db, sets)
	check(err)
	log.Println("Imported sets.")

	log.Println("Downloading bulk data entry...")
	allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards)
	check(err)
	log.Println("Downloaded bulk data entry.")

	updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
	check(err)
	updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second) // Remove the fractional seconds component from the timestamp

	log.Println("Checking for redownload...")
	updated, err := cardsUpdatedSinceCache(db, updatedAtTimestamp)
	check(err)

	if updated {
		log.Printf("Bulk data has been updated since last cache, redownloading.")
		err = cacheAllCardsFile(db, allCardsBulkData.DownloadUri, updatedAtTimestamp)
		check(err)
	} else {
		log.Printf("Bulk data has not been updated. Skipping download.")
	}

	log.Printf("Unmarsaling file into slice...")
	allCardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME)
	check(err)

	var allCards []types.Card
	err = json.Unmarshal(allCardsBytes, &allCards)
	check(err)
	log.Printf("Unmarshaled file.")

	log.Printf("Importing card data into database...")
	err = importCards(db, allCards)
	check(err)
	log.Printf("Imported card data.")
}