Refactor import/update code

sevenkeys/database/cachetimestamp.go

@@ -1,6 +1,9 @@
 package database
 
-import "database/sql"
+import (
+	"database/sql"
+	"time"
+)
 
 const CacheTypeAllCardsBulkData = "AllCardsBulkData"
 
@@ -12,3 +15,23 @@ func GetCacheTimestampByType(db *sql.DB, cacheType string) (string, error) {
 
 	return timestamp, err
 }
+
+func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error {
+	query := `INSERT INTO CacheTimestamp (CacheType, Stamp)
+		VALUES (?, ?)
+		ON DUPLICATE KEY
+		UPDATE Stamp = ?;`
+
+	insertOrUpdate, err := db.Prepare(query)
+	defer insertOrUpdate.Close()
+	if err != nil {
+		return err
+	}
+
+	_, err = insertOrUpdate.Exec(cacheType, stamp, stamp)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

sevenkeys/database/card.go

@@ -0,0 +1,39 @@
+package database
+
+import (
+	"database/sql"
+)
+
+type CardPrinting struct {
+	Id              int
+	Name            string
+	SetCode         string
+	IsFoil          bool
+	IsPromo         bool
+	CollectorNumber string
+	Language        string
+}
+
+func InsertCardPrinting(db *sql.DB, cardPrinting CardPrinting) error {
+	query := `INSERT INTO CardPrinting (
+		Name,
+		SetCode,
+		IsFoil,
+		IsPromo,
+		CollectorNumber,
+		Language)
+		VALUES (?, ?, ?, ?, ?, ?);`
+
+	insert, err := db.Prepare(query)
+	defer insert.Close()
+	if err != nil {
+		return err
+	}
+
+	_, err = insert.Exec(cardPrinting.Name, cardPrinting.SetCode, cardPrinting.IsFoil, cardPrinting.IsPromo, cardPrinting.CollectorNumber, cardPrinting.Language)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

sevenkeys/database/operations/inserts.go

@@ -2,70 +2,8 @@ package operations
 
 import (
 	"database/sql"
-	"sevenkeys/logic/scryfall/types"
-	"time"
 )
 
-func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error {
-	query := `INSERT INTO CacheTimestamp (CacheType, Stamp)
-		VALUES (?, ?)
-		ON DUPLICATE KEY
-		UPDATE Stamp = ?;`
-
-	insertOrUpdate, err := db.Prepare(query)
-	defer insertOrUpdate.Close()
-	if err != nil {
-		return err
-	}
-
-	_, err = insertOrUpdate.Exec(cacheType, stamp, stamp)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// TODO: There's really no need for this to be an "upsert", we can just ignore sets that we already have
-func InsertOrUpdateSet(db *sql.DB, set types.Set) error {
-	query := `INSERT INTO ExpansionSet (SetCode, Name, CardCount, IconSvgUri)
-		VALUES (?, ?, ?, ?)
-		ON DUPLICATE KEY
-		Update Name = ?, CardCount = ?, IconSvgUri = ?;`
-
-	insertOrUpdate, err := db.Prepare(query)
-	defer insertOrUpdate.Close()
-	if err != nil {
-		return err
-	}
-
-	_, err = insertOrUpdate.Exec(set.Code, set.Name, set.CardCount, set.IconSvgUri, set.Name, set.CardCount, set.IconSvgUri)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func InsertCard(db *sql.DB, card types.Card) error {
-	query := `INSERT IGNORE INTO CardPrinting
-		(Id, Name, SetCode, HasFoil, HasNonFoil, IsReserved, IsRacist, IsPromo, CollectorNumber, Language)
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`
-
-	insert, err := db.Prepare(query)
-	defer insert.Close()
-	if err != nil {
-		return err
-	}
-
-	_, err = insert.Exec(card.Id, card.Name, card.Set, card.Foil, card.NonFoil, card.Reserved, card.ContentWarning, card.Promo, card.CollectorNumber, card.Language)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func InsertCardStorageLocation(db *sql.DB, cardPrintingId string, isFoil bool, storageBox string, source string) error {
 	var lastPosition int
 	getLastPositionQuery := `SELECT Position FROM CardStorageLocation WHERE StorageBox = ? ORDER BY Position DESC LIMIT 1;`

sevenkeys/database/set.go

@@ -0,0 +1,23 @@
+package database
+
+import (
+	"database/sql"
+	"sevenkeys/logic/scryfall"
+)
+
+func InsertSet(db *sql.DB, set scryfall.Set) error {
+	query := `INSERT INTO ExpansionSet (SetCode, Name, CardCount, IconSvgUri) VALUES (?, ?, ?, ?);`
+
+	insert, err := db.Prepare(query)
+	defer insert.Close()
+	if err != nil {
+		return err
+	}
+
+	_, err = insert.Exec(set.Code, set.Name, set.CardCount, set.IconSvgUri)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

sevenkeys/database/sql/createdb.sql

@@ -15,25 +15,22 @@ CREATE TABLE IF NOT EXISTS ExpansionSet (
 );
 
 CREATE TABLE IF NOT EXISTS CardPrinting (
-	Id VARCHAR(36) PRIMARY KEY, -- GUID
+	Id INT AUTO_INCREMENT PRIMARY KEY,
 	Name VARCHAR(150) NOT NULL,
 	SetCode VARCHAR(6) NOT NULL,
 	FOREIGN KEY (SetCode) REFERENCES ExpansionSet(SetCode),
-	HasFoil BOOLEAN NOT NULL,
+	IsFoil BOOLEAN NOT NULL,
-	HasNonFoil BOOLEAN NOT NULL,
-	IsReserved BOOLEAN NOT NULL,
-	IsRacist BOOLEAN NOT NULL,
 	IsPromo BOOLEAN NOT NULL,
 	CollectorNumber VARCHAR(10) NOT NULL,
 	Language VARCHAR(3) NOT NULL
 );
 
-CREATE TABLE IF NOT EXISTS CardStorageLocation (
+-- CREATE TABLE IF NOT EXISTS CardStorageLocation (
-	Id INT AUTO_INCREMENT PRIMARY KEY,
+	-- Id INT AUTO_INCREMENT PRIMARY KEY,
-	CardPrintingId VARCHAR(36) NOT NULL,
+	-- CardPrintingId VARCHAR(36) NOT NULL,
-	FOREIGN KEY (CardPrintingId) REFERENCES CardPrinting(Id),
+	-- FOREIGN KEY (CardPrintingId) REFERENCES CardPrinting(Id),
-	IsFoil BOOLEAN NOT NULL,
+	-- IsFoil BOOLEAN NOT NULL,
-	StorageBox VARCHAR(20) NOT NULL,
+	-- StorageBox VARCHAR(20) NOT NULL,
-	Source VARCHAR(100) NULL,
+	-- Source VARCHAR(100) NULL,
-	Position INT NOT NULL
+	-- Position INT NOT NULL
-);
+-- );

sevenkeys/go.mod

@@ -6,6 +6,7 @@ require github.com/go-sql-driver/mysql v1.8.1
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	gotest.tools/v3 v3.5.1 // indirect
 )

sevenkeys/go.sum

@@ -1,5 +1,7 @@
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0 h1:meiLwrW6ukHHehydhoDxVHdQKQe7TFgEpH0A0hHBAWs=
+github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0/go.mod h1:ZH8V0509n2OSZLMYTMHzcy4hqUB+rG8ghK1zsP4i5gE=
 github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=

sevenkeys/importer.go

@@ -1,175 +0,0 @@
-package main
-
-import (
-	"database/sql"
-	"encoding/json"
-	"io"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"sevenkeys/database/entities"
-	"sevenkeys/database/operations"
-	"sevenkeys/logic/scryfall/types"
-	"time"
-)
-
-const CACHE_DIR string = "cache"
-
-const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/"
-const SET_ICON_FILE_EXTENSION string = ".svg"
-
-const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json"
-
-const GAME_PAPER = "paper"
-
-func check(err error) {
-	if err != nil {
-		log.Fatal(err)
-	}
-}
-
-func createCacheDirectories() error {
-	err := os.Mkdir(CACHE_DIR, os.ModePerm)
-	if err != nil && !os.IsExist(err) {
-		return err
-	}
-
-	err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm)
-	if err != nil && !os.IsExist(err) {
-		return err
-	}
-
-	return nil
-}
-
-func importSets(db *sql.DB, sets []types.Set) error {
-	for _, set := range sets {
-		// We're only interested in paper cards, so skip importing
-		// any sets that were only released in a video game
-		if set.Digital {
-			continue
-		}
-
-		log.Println("Importing " + set.Code)
-		err := operations.InsertOrUpdateSet(db, set)
-		if err != nil {
-			return err
-		}
-
-		log.Println("Downloading logo for " + set.Code)
-		response, err := http.Get(set.IconSvgUri)
-		defer response.Body.Close()
-		if err != nil {
-			return err
-		}
-
-		iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION
-		iconFile, err := os.Create(iconFilename)
-		if err != nil {
-			return err
-		}
-
-		io.Copy(iconFile, response.Body)
-		log.Println("Finished importing " + set.Code)
-	}
-
-	return nil
-}
-
-func cacheAllCardsFile(db *sql.DB, uri string, updatedAtTimestamp time.Time) error {
-	log.Printf("Downloading bulk card data...")
-	bulkCardsResponse, err := http.Get(uri)
-	if err != nil {
-		return err
-	}
-	log.Printf("Downloaded bulk card data.")
-
-	log.Printf("Writing card data to cache file...")
-	cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
-	if err != nil {
-		return err
-	}
-
-	defer bulkCardsResponse.Body.Close()
-	defer cacheFile.Close()
-	io.Copy(cacheFile, bulkCardsResponse.Body)
-	log.Printf("Cache file written.")
-
-	log.Printf("Recording timestamp...")
-	err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
-	if err != nil {
-		return err
-	}
-	log.Printf("Timestamp recorded.")
-
-	return nil
-}
-
-func importCards(db *sql.DB, cards []types.Card) error {
-	for _, card := range cards {
-		// We're only interested in paper cards, so skip cards or printings of a card which
-		// aren't available in paper
-		if !isPaper(card) {
-			continue
-		}
-
-		err := operations.InsertCard(db, card)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func isPaper(card types.Card) bool {
-	var paper bool = false
-
-	for _, game := range card.Games {
-		if game == GAME_PAPER {
-			paper = true
-		}
-	}
-
-	return paper
-}
-
-func main() {
-	log.Println("Creating cache directories...")
-	err := createCacheDirectories()
-	check(err)
-	log.Println("Created cache directories.")
-
-	log.Println("Downloading set data from Scryfall...")
-	sets, err := methods.GetSets()
-	check(err)
-	log.Println("Downloaded set data.")
-
-	log.Println("Importing set data...")
-	err = importSets(db, sets)
-	check(err)
-	log.Println("Imported sets.")
-
-	if updated {
-		log.Printf("Bulk data has been updated since last cache, redownloading.")
-		err = cacheAllCardsFile(db, allCardsBulkData.DownloadUri, updatedAtTimestamp)
-		check(err)
-	} else {
-		log.Printf("Bulk data has not been updated. Skipping download.")
-	}
-
-	log.Printf("Unmarsaling file into slice...")
-	allCardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME)
-	check(err)
-
-	var allCards []types.Card
-	err = json.Unmarshal(allCardsBytes, &allCards)
-	check(err)
-	log.Printf("Unmarshaled file.")
-
-	log.Printf("Importing card data into database...")
-	err = importCards(db, allCards)
-	check(err)
-	log.Printf("Imported card data.")
-}

sevenkeys/logic/scryfall/bulkdata.go

@@ -5,19 +5,21 @@ import (
 	"errors"
 	"io"
 	"net/http"
+	"time"
 )
 
 type BulkData struct {
-	Id              string `json:"id"`
+	Id              string    `json:"id"`
-	Uri             string `json:"uri"`
+	Uri             string    `json:"uri"`
-	Type            string `json:"type"`
+	Type            string    `json:"type"`
-	Name            string `json:"name"`
+	Name            string    `json:"name"`
-	Description     string `json:"description"`
+	Description     string    `json:"description"`
-	DownloadUri     string `json:"download_uri"`
+	DownloadUri     string    `json:"download_uri"`
-	UpdatedAt       string `json:"updated_at"`
+	UpdatedAt       string    `json:"updated_at"`
-	Size            int    `json:"size"`
+	UpdatedAtTime   time.Time `json:"ignore"`
-	ContentType     string `json:"content_type"`
+	Size            int       `json:"size"`
-	ContentEncoding string `json:"content_encoding"`
+	ContentType     string    `json:"content_type"`
+	ContentEncoding string    `json:"content_encoding"`
 }
 
 const BULK_DATA_URI = "https://api.scryfall.com/bulk-data"
@@ -50,5 +52,12 @@ func GetBulkDataByType(bulkDataType string) (BulkData, error) {
 		return BulkData{}, err
 	}
 
+	bulkData.UpdatedAtTime, err = time.Parse(ScryfallTimestampFormat, bulkData.UpdatedAt)
+	if err != nil {
+		return BulkData{}, err
+	}
+	// Round to the nearest second; this is so that comparison with the timestamp stored in the database works as intended
+	bulkData.UpdatedAtTime = bulkData.UpdatedAtTime.Truncate(time.Second)
+
 	return bulkData, nil
 }

sevenkeys/logic/scryfall/card.go

@@ -1,14 +1,11 @@
 package scryfall
 
 type Card struct {
-	Id              string   `json:"id"` // GUID
 	Name            string   `json:"name"`
 	Set             string   `json:"set"`
 	Games           []string `json:"games"`
 	Foil            bool     `json:"foil"`
 	NonFoil         bool     `json:"nonfoil"`
-	Reserved        bool     `json:"reserved"`
-	ContentWarning  bool     `json:"content_warning,omitempty"`
 	Promo           bool     `json:"promo"`
 	CollectorNumber string   `json:"collector_number"`
 	Language        string   `json:"lang"`

sevenkeys/logic/scryfall/sets.go

@@ -23,7 +23,7 @@ type Set struct {
 
 const SETS_API_URL string = "https://api.scryfall.com/sets"
 
-func GetSets() ([]Set, error) {
+func GetAllSets() ([]Set, error) {
 	response, err := http.Get(SETS_API_URL)
 	if err != nil {
 		return []Set{}, nil

sevenkeys/logic/update.go

@@ -2,14 +2,27 @@ package logic
 
 import (
 	"database/sql"
+	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
 	"sevenkeys/database"
 	"sevenkeys/logic/scryfall"
 	"strings"
 	"time"
+
+	sqlerr "github.com/go-mysql/errors"
 )
 
-func CheckForUpdates(db *sql.DB) (bool, error) {
+const GAME_PAPER = "paper"
+
+func CheckForUpdates(db *sql.DB, bulkData scryfall.BulkData) (bool, error) {
+	// TODO: We also want to update if:
+	// - there is no cached version of the all-cards.json file or it is empty
+	// - The set icon files are missing
 	cachedFileTimestampStr, err := database.GetCacheTimestampByType(db, database.CacheTypeAllCardsBulkData)
 	if err == sql.ErrNoRows {
 		return true, nil
@@ -19,18 +32,7 @@ func CheckForUpdates(db *sql.DB) (bool, error) {
 
 	cachedFileTimestamp, err := time.Parse("2006-01-02 15:04:05", cachedFileTimestampStr)
 
-	allCardsBulkData, err := scryfall.GetBulkDataByType(scryfall.BulkDataTypeAllCards)
+	return bulkData.UpdatedAtTime.After(cachedFileTimestamp), nil
-	if err != nil {
-		return false, err
-	}
-
-	bulkCardsUpdatedTimestamp, err := time.Parse(scryfall.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
-	if err != nil {
-		return false, err
-	}
-	bulkCardsUpdatedTimestamp = bulkCardsUpdatedTimestamp.Truncate(time.Second)
-
-	return bulkCardsUpdatedTimestamp.After(cachedFileTimestamp), nil
 }
 
 func ConfirmUpdate() bool {
@@ -40,3 +42,162 @@ func ConfirmUpdate() bool {
 
 	return strings.ToUpper(response) == "Y"
 }
+
+func UpdateSets(db *sql.DB) error {
+	sets, err := scryfall.GetAllSets()
+	if err != nil {
+		return err
+	}
+
+	for _, set := range sets {
+		// We're only interested in paper cards, so skip importing
+		// any sets that were only released in a video game
+		if set.Digital {
+			continue
+		}
+
+		err := database.InsertSet(db, set)
+		// If we already have this set in the database, then we can just skip it
+		if ok, insertErr := sqlerr.Error(err); ok {
+			if insertErr == sqlerr.ErrDupeKey {
+				continue
+			}
+		}
+
+		if err != nil {
+			return err
+		}
+
+		response, err := http.Get(set.IconSvgUri)
+		defer response.Body.Close()
+		if err != nil {
+			return nil
+		}
+
+		iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION
+		iconFile, err := os.Create(iconFilename)
+		if err != nil {
+			return err
+		}
+
+		io.Copy(iconFile, response.Body)
+	}
+
+	fmt.Println("Sets updated.")
+	return nil
+}
+
+func cacheBulkCardsFile(db *sql.DB, bulkData scryfall.BulkData) error {
+	bulkCardsResponse, err := http.Get(bulkData.DownloadUri)
+	defer bulkCardsResponse.Body.Close()
+	if err != nil {
+		return err
+	}
+
+	cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
+	defer cacheFile.Close()
+	if err != nil {
+		return err
+	}
+
+	io.Copy(cacheFile, bulkCardsResponse.Body)
+
+	err = database.InsertOrUpdateCacheTimestampByType(db, database.CacheTypeAllCardsBulkData, bulkData.UpdatedAtTime)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func isPaper(card scryfall.Card) bool {
+	var paper bool = false
+
+	for _, game := range card.Games {
+		if game == GAME_PAPER {
+			paper = true
+		}
+	}
+
+	return paper
+}
+
+func getCardPrintings(card scryfall.Card) []database.CardPrinting {
+	var printings []database.CardPrinting
+
+	if card.Foil {
+		printings = append(printings, database.CardPrinting{
+			Name:            card.Name,
+			SetCode:         card.Set,
+			IsFoil:          true,
+			IsPromo:         card.Promo,
+			CollectorNumber: card.CollectorNumber,
+			Language:        card.Language,
+		})
+	}
+
+	if card.NonFoil {
+		printings = append(printings, database.CardPrinting{
+			Name:            card.Name,
+			SetCode:         card.Set,
+			IsFoil:          false,
+			IsPromo:         card.Promo,
+			CollectorNumber: card.CollectorNumber,
+			Language:        card.Language,
+		})
+	}
+
+	return printings
+}
+
+func UpdateCards(db *sql.DB, bulkData scryfall.BulkData) error {
+	log.Println("Caching bulk cards file")
+	err := cacheBulkCardsFile(db, bulkData)
+	if err != nil {
+		return err
+	}
+	log.Println("Cached bulk cards file")
+
+	log.Println("Reading cached file")
+	cardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME)
+	if err != nil {
+		return err
+	}
+	log.Println("Read cached file")
+
+	log.Println("Unmarshaling JSON")
+	var cards []scryfall.Card
+	err = json.Unmarshal(cardsBytes, &cards)
+	if err != nil {
+		return err
+	}
+	log.Println("Unmarshaled JSON")
+
+	log.Println("INSERTing cards")
+	for _, card := range cards {
+		// We're only interested in paper cards, so skip cards or printings of a card which
+		// aren't available in paper
+		if !isPaper(card) {
+			continue
+		}
+
+		cardPrintings := getCardPrintings(card)
+
+		for _, cardPrinting := range cardPrintings {
+			err := database.InsertCardPrinting(db, cardPrinting)
+			// If we already have this card in the database, then we can just skip it
+			if ok, insertErr := sqlerr.Error(err); ok {
+				if insertErr == sqlerr.ErrDupeKey {
+					continue
+				}
+			}
+		}
+
+		if err != nil {
+			return err
+		}
+	}
+	log.Println("INSERTed cards")
+
+	return nil
+}

sevenkeys/main.go

@@ -4,23 +4,36 @@ import (
 	"fmt"
 	"sevenkeys/database"
 	"sevenkeys/logic"
+	"sevenkeys/logic/scryfall"
 )
 
 func main() {
 	db := database.GetDatabaseFromConfig("config.json")
 
 	fmt.Println("Checking for updates...")
-	needsUpdate, err := logic.CheckForUpdates(db)
+	bulkData, err := scryfall.GetBulkDataByType(scryfall.BulkDataTypeAllCards)
+	logic.Check(err)
+
+	needsUpdate, err := logic.CheckForUpdates(db, bulkData)
 	logic.Check(err)
 
 	if needsUpdate {
 		fmt.Println("Update required.")
+
 		if logic.ConfirmUpdate() {
-			fmt.Println("User authorized update")
+			fmt.Println("Running update...")
-			/*
+
-				err = logic.RunScryfallUpdate()
+			logic.CreateCacheDirectories()
-				logic.Check(err)
+
-			*/
+			err = logic.UpdateSets(db)
+			logic.Check(err)
+
+			err = logic.UpdateCards(db, bulkData)
+			logic.Check(err)
 		}
+
+		fmt.Println("Update finished.")
+	} else {
+		fmt.Println("No update required.")
 	}
 }