Import Set data from Scryfall

sevenkeys/Makefile

@@ -1,4 +1,8 @@
-dbup:
+removedb:
+	mysql --user=root --password=$(shell pass show sevenkeys/mysql) <sql/removedb.sql
+createdb:
+	mysql --user=root --password=$(shell pass show sevenkeys/mysql) <sql/createdb.sql
+dbup: createdb
 	go run cmd/dbup/main.go
 importdata: dbup
 	go run cmd/importdata/main.go

sevenkeys/cmd/dbup/main.go

@@ -1,32 +1,10 @@
 package main
 
 import (
-	"encoding/json"
-	"log"
-	"os"
-
 	"sevenkeys/database"
-
-	"github.com/go-sql-driver/mysql"
 )
 
-func readConfigFromFile(filename string) mysql.Config {
-	bytes, err := os.ReadFile(filename)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	var config mysql.Config
-	err = json.Unmarshal(bytes, &config)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return config
-}
-
 func main() {
-	config := readConfigFromFile("config.json")
+	db := database.GetDatabaseFromConfig("config.json")
-	db := database.GetDatabase(config)
 	database.CreateDatabaseSchema(db)
 }

sevenkeys/cmd/importdata/main.go

@@ -2,43 +2,29 @@ package main
 
 import (
 	"database/sql"
-	"fmt"
 	"log"
+	"sevenkeys/database"
+	"sevenkeys/scryfall"
 )
 
-func populateColorTable(db *sql.DB) {
+func importSets(db *sql.DB) {
-	query := `INSERT INTO Color (Name, Abbreviation) VALUES (?, ?);`
+	setList, err := scryfall.GetSets()
-
-	insert, err := db.Prepare(query)
-	defer insert.Close()
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	colors := map[string]string{
+	err = database.InsertSets(db, setList)
-		"White": "W",
+	if err != nil {
-		"Blue":  "U",
+		log.Fatal(err)
-		"Black": "B",
-		"Red":   "R",
-		"Green": "G",
-	}
-	for name, abbrev := range colors {
-		// TODO: Don't import a color if it's already in the database
-		result, err := insert.Exec(name, abbrev)
-		rowsAffected, err := result.RowsAffected()
-		if err != nil || rowsAffected != 1 {
-			log.Fatal(err)
-		}
 	}
 }
 
 func main() {
-	// Import colors
+	db := database.GetDatabaseFromConfig("config.json")
-	// populateColorTable(db)
+	importColors(db)
-	// Import sets
+	importSets(db)
 	// Import keywords
 	// Import formats
 	// Import artists
 	// Import gamepieces and printings
-	fmt.Println("Hello, world from importdata")
 }

sevenkeys/database/entities/expansion_set.go

@@ -0,0 +1,18 @@
+package entities
+
+type ExpansionSet struct {
+	Id            int
+	SetCode       string
+	Name          string
+	SetType       string
+	ReleasedAt    string
+	BlockCode     string
+	Block         string
+	ParentSetCode string
+	CardCount     int
+	PrintedSize   int
+	Digital       bool
+	FoilOnly      bool
+	NonfoilOnly   bool
+	IconSvgUri    string
+}

sevenkeys/database/get_database.go

@@ -0,0 +1,40 @@
+package database
+
+import (
+	"database/sql"
+	"encoding/json"
+	"log"
+	"os"
+
+	"github.com/go-sql-driver/mysql"
+)
+
+func readConfigFromFile(filename string) mysql.Config {
+	bytes, err := os.ReadFile(filename)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var config mysql.Config
+	err = json.Unmarshal(bytes, &config)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return config
+}
+
+func GetDatabaseFromConfig(filename string) *sql.DB {
+	config := readConfigFromFile(filename)
+
+	db, err := sql.Open("mysql", config.FormatDSN())
+
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err = db.Ping(); err != nil {
+		log.Fatal(err)
+	}
+
+	return db
+}

sevenkeys/database/imports/color.go

@@ -0,0 +1,29 @@
+package database
+
+/*
+func populateColorTable(db *sql.DB) {
+	query := `INSERT INTO Color (Name, Abbreviation) VALUES (?, ?);`
+
+	insert, err := db.Prepare(query)
+	defer insert.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	colors := map[string]string{
+		"White": "W",
+		"Blue":  "U",
+		"Black": "B",
+		"Red":   "R",
+		"Green": "G",
+	}
+	for name, abbrev := range colors {
+		// TODO: Don't import a color if it's already in the database
+		result, err := insert.Exec(name, abbrev)
+		rowsAffected, err := result.RowsAffected()
+		if err != nil || rowsAffected != 1 {
+			log.Fatal(err)
+		}
+	}
+}
+*/

sevenkeys/database/imports/expansionset.go

@@ -0,0 +1,58 @@
+package database
+
+import (
+	"database/sql"
+	"log"
+	"sevenkeys/scryfall"
+)
+
+func CheckSetExists(db *sql.DB, setCode string) bool {
+	query := "SELECT Id FROM ExpansionSet WHERE SetCode = ?;"
+
+	var setId int
+	row := db.QueryRow(query, setCode)
+	err := row.Scan(&setId)
+
+	if err == sql.ErrNoRows {
+		return false
+	} else if err != nil {
+		log.Fatal(err)
+	}
+
+	return true
+}
+
+func insertSet(db *sql.DB, set scryfall.Set) error {
+	query := "INSERT INTO ExpansionSet (SetCode, Name, SetType, ReleasedAt, BlockCode, Block, ParentSetCode, CardCount, PrintedSize, Digital, FoilOnly, NonfoilOnly, IconSvgUri) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"
+
+	insert, err := db.Prepare(query)
+	defer insert.Close()
+	if err != nil {
+		return err
+	}
+
+	if CheckSetExists(db, set.Code) {
+		log.Println("Skipping " + set.Code + ", already in database")
+		return nil
+	}
+
+	_, err = insert.Exec(set.Code, set.Name, set.SetType, set.ReleasedAt, set.BlockCode, set.Block, set.ParentSetCode, set.CardCount, set.PrintedSize, set.Digital, set.FoilOnly, set.NonfoilOnly, set.IconSvgUri)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func InsertSets(db *sql.DB, setList scryfall.SetList) error {
+	for index := range setList.Data {
+		set := setList.Data[index]
+
+		err := insertSet(db, set)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

sevenkeys/database/setup.go

@@ -3,25 +3,8 @@ package database
 import (
 	"database/sql"
 	"log"
-
-	"github.com/go-sql-driver/mysql"
 )
 
-var db *sql.DB
-
-func GetDatabase(config mysql.Config) *sql.DB {
-	db, err := sql.Open("mysql", config.FormatDSN())
-
-	if err != nil {
-		log.Fatal(err)
-	}
-	if err = db.Ping(); err != nil {
-		log.Fatal(err)
-	}
-
-	return db
-}
-
 func createGamepieceTable(db *sql.DB) {
 	query := `CREATE TABLE IF NOT EXISTS Gamepiece (
 		Id INT AUTO_INCREMENT PRIMARY KEY,
@@ -184,10 +167,19 @@ func createGamepieceFormatLegalityTable(db *sql.DB) {
 func createSetTable(db *sql.DB) {
 	query := `CREATE TABLE IF NOT EXISTS ExpansionSet (
 		Id INT AUTO_INCREMENT PRIMARY KEY,
-		SetCode VARCHAR(5) NOT NULL,
+		SetCode VARCHAR(6) NOT NULL,
-		Name VARCHAR(40) NOT NULL,
+		Name VARCHAR(60) NOT NULL,
+		SetType VARCHAR(20) NOT NULL,
+		ReleasedAt DATETIME NOT NULL,
+		BlockCode VARCHAR(10) NULL,
+		Block VARCHAR(40) NULL,
+		ParentSetCode VARCHAR(5) NULL,
+		CardCount INT NOT NULL,
+		PrintedSize INT NULL,
 		Digital BOOLEAN NOT NULL,
-		SetType VARCHAR(20) NOT NULL
+		FoilOnly BOOLEAN NOT NULL,
+		NonfoilOnly BOOLEAN NOT NULL,
+		IconSvgUri VARCHAR(60) NOT NULL
 	);`
 
 	_, err := db.Exec(query)

sevenkeys/scryfall/set.go

@@ -0,0 +1,65 @@
+package scryfall
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+)
+
+type Set struct {
+	Object        string `json:"object"`
+	Id            string `json:"id"` // Use UUID package?
+	Code          string `json:"code"`
+	MtgoCode      string `json:"mtgo_code"`
+	ArenaCode     string `json:"arena_code"`
+	TcgplayerId   int    `json:"tcgplayer_id"`
+	Name          string `json:"name"`
+	SetType       string `json:"set_type"`
+	ReleasedAt    string `json:"released_at"`
+	BlockCode     string `json:"block_code"`
+	Block         string `json:"block"`
+	ParentSetCode string `json:"parent_set_code"`
+	CardCount     int    `json:"card_count"`
+	PrintedSize   int    `json:"printed_size"`
+	Digital       bool   `json:"digital"`
+	FoilOnly      bool   `json:"foil_only"`
+	NonfoilOnly   bool   `json:"nonfoil_only"`
+	ScryfallUri   string `json:"scryfall_uri"`
+	Uri           string `json:"uri"`
+	IconSvgUri    string `json:"icon_svg_uri"`
+	SearchUri     string `json:"search_uri"`
+}
+
+type SetList struct {
+	Object  string `json:"object"`
+	HasMore bool   `json:"has_more"`
+	Data    []Set  `json:"data"`
+}
+
+var SETS_API string = "https://api.scryfall.com/sets"
+
+func GetSets() (SetList, error) {
+	response, err := http.Get(SETS_API)
+	if err != nil {
+		return SetList{}, err
+	}
+
+	defer response.Body.Close()
+	if response.StatusCode == http.StatusOK {
+		setsBytes, err := io.ReadAll(response.Body)
+		if err != nil {
+			return SetList{}, err
+		}
+
+		var setList SetList
+		err = json.Unmarshal(setsBytes, &setList)
+		if err != nil {
+			return SetList{}, err
+		}
+
+		return setList, nil
+	}
+
+	return SetList{}, errors.New("scryfall: failed to get all sets")
+}

sevenkeys/sql/createdb.sql

@@ -0,0 +1 @@
+CREATE DATABASE IF NOT EXISTS sevenkeys;

sevenkeys/sql/removedb.sql

@@ -0,0 +1 @@
+DROP DATABASE IF EXISTS sevenkeys;