Skip digital-only cards, close insert statements

a_scanner_darkly/.gitignore

@@ -0,0 +1,5 @@
+cmake_install.cmake
+CMakeCache.txt
+CMakeFiles/*
+Makefile
+AScannerDarkly

a_scanner_darkly/CMakeLists.txt

@@ -0,0 +1,6 @@
+cmake_minimum_required(VERSION 3.2)
+project( AScannerDarkly )
+find_package( OpenCV REQUIRED )
+include_directories( ${OpenCV_INCLUDE_DIRS} )
+add_executable( AScannerDarkly main.cpp )
+target_link_libraries( AScannerDarkly ${OpenCV_LIBS} )

a_scanner_darkly/main.cpp

@@ -0,0 +1,39 @@
+#include <opencv2/opencv.hpp>
+
+const std::string WINDOW_NAME = "A Scanner Darkly";
+const std::string LOWER_THRESHOLD_TRACKBAR_NAME = "Canny: Lower Threshold";
+const std::string UPPER_THRESHOLD_TRACKBAR_NAME = "Canny: Upper Threshold";
+
+int g_Canny_lower_threshold = 110;
+int g_Canny_upper_threshold = 300;
+
+int main(int argc, char** argv ) {
+	cv::VideoCapture cap;
+	cap.open(0);
+
+	if (!cap.isOpened()) {
+		std::cerr << "Couldn't open capture" << std::endl;
+		return -1;
+	}
+
+	cv::namedWindow(WINDOW_NAME);
+	cv::createTrackbar(LOWER_THRESHOLD_TRACKBAR_NAME, WINDOW_NAME, &g_Canny_lower_threshold, 1000, NULL);
+	cv::createTrackbar(UPPER_THRESHOLD_TRACKBAR_NAME, WINDOW_NAME, &g_Canny_upper_threshold, 1000, NULL);
+
+	cv::Mat frame, grayscaleFrame, cannyFrame;
+	while (true) {
+		cap >> frame;
+
+		cv::cvtColor(frame, grayscaleFrame, cv::COLOR_BGR2GRAY);
+		cv::Canny(grayscaleFrame, cannyFrame, g_Canny_lower_threshold, g_Canny_upper_threshold);
+
+		cv::imshow(WINDOW_NAME, cannyFrame);
+
+		char c = (char)cv::waitKey(33);
+		if (c == 27) {
+			break;
+		}
+	}
+
+	cv::destroyWindow(WINDOW_NAME);
+}

sevenkeys/.gitignore

@@ -1 +1,2 @@
+cache/
 cmd/database/config.json

sevenkeys/Makefile

@@ -1,10 +1,8 @@
-removedb:
-	mysql --user=root --password=$(shell pass show sevenkeys/mysql) <sql/removedb.sql
-
 createdb:
 	mysql --user=root --password=$(shell pass show sevenkeys/mysql) <sql/createdb.sql
-importcards: createdb
+import: createdb
-	go run cmd/importcards/main.go
+	go run main.go
 
-getdecks:
+clean:
-	go run cmd/getdecks/main.go
+	mysql --user=root --password=$(shell pass show sevenkeys/mysql) <sql/removedb.sql
+	rm -rf cache/

sevenkeys/cmd/getdecks/main.go

@@ -1,18 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"sevenkeys/mtggoldfish"
-)
-
-func main() {
-	params := mtggoldfish.DeckSearchParameters{
-		Format:                 "pauper",
-		IncludeTournamentDecks: true,
-		StartDate:              "04/14/2024",
-		EndDate:                "04/28/2024",
-	}
-
-	decks, _ := mtggoldfish.DeckSearch(params)
-	fmt.Println(decks)
-}

sevenkeys/cmd/importcards/main.go

@@ -1,13 +0,0 @@
-package main
-
-import (
-	"sevenkeys/database"
-	"sevenkeys/database/operations"
-	"sevenkeys/scryfall/methods"
-)
-
-func main() {
-	db := database.GetDatabaseFromConfig("config.json")
-	allCards := methods.GetAllCards()
-	operations.InsertCards(db, allCards)
-}

sevenkeys/database/entities/cachetimestamp.go

@@ -0,0 +1,3 @@
+package entities
+
+const CacheTypeAllCardsBulkData = "AllCardsBulkData"

sevenkeys/database/operations/inserts.go

@@ -2,57 +2,66 @@ package operations
 
 import (
 	"database/sql"
-	"errors"
-	"log"
 	"sevenkeys/scryfall/types"
+	"time"
 )
 
-var ErrGamepieceExists error = errors.New("Gamepiece already exists in database")
+func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error {
+	query := `INSERT INTO CacheTimestamp (CacheType, Stamp)
+		VALUES (?, ?)
+		ON DUPLICATE KEY
+		UPDATE Stamp = ?;`
 
-func insertGamepiece(db *sql.DB, cardName string) (int, error) {
+	insertOrUpdate, err := db.Prepare(query)
-	gamepieceInsertQuery := "INSERT INTO Gamepiece (Name) VALUES (?);"
+	defer insertOrUpdate.Close()
-	gamepieceInsert, err := db.Prepare(gamepieceInsertQuery)
-	defer gamepieceInsert.Close()
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
-	existingGamepiece, err := GetGamepieceByName(db, cardName)
+	_, err = insertOrUpdate.Exec(cacheType, stamp, stamp)
-	if err != sql.ErrNoRows {
-		return existingGamepiece.Id, ErrGamepieceExists
-	}
-
-	result, err := gamepieceInsert.Exec(cardName)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
-	gamepieceId, err := result.LastInsertId()
+	return nil
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return int(gamepieceId), nil
 }
 
-func insertCardPrinting(db *sql.DB, gamepieceId int, setCode string, imageUrl string) {
+// TODO: There's really no need for this to be an "upsert", we can just ignore sets that we already have
-	cardPrintingInsertQuery := "INSERT INTO CardPrinting (GamepieceId, SetCode, ImageUrl) VALUES (?, ?, ?);"
+func InsertOrUpdateSet(db *sql.DB, set types.Set) error {
-	cardPrintingInsert, err := db.Prepare(cardPrintingInsertQuery)
+	query := `INSERT INTO ExpansionSet (SetCode, Name, CardCount, IconSvgUri)
-	defer cardPrintingInsert.Close()
+		VALUES (?, ?, ?, ?)
+		ON DUPLICATE KEY
+		Update Name = ?, CardCount = ?, IconSvgUri = ?;`
+
+	insertOrUpdate, err := db.Prepare(query)
+	defer insertOrUpdate.Close()
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
-	_, err = cardPrintingInsert.Exec(gamepieceId, setCode, imageUrl)
+	_, err = insertOrUpdate.Exec(set.Code, set.Name, set.CardCount, set.IconSvgUri, set.Name, set.CardCount, set.IconSvgUri)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
+
+	return nil
 }
 
-func InsertCards(db *sql.DB, cards []types.Card) {
+func InsertCard(db *sql.DB, card types.Card) error {
-	for index := range cards {
+	query := `INSERT IGNORE INTO CardPrinting
-		card := cards[index]
+		(Id, Name, SetCode, HasFoil, HasNonFoil, IsReserved, IsRacist, IsPromo, CollectorNumber, Language)
-		gamepieceId, _ := insertGamepiece(db, card.Name)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`
-		insertCardPrinting(db, gamepieceId, card.Set, card.ImageUris["png"])
+
+	insert, err := db.Prepare(query)
+	defer insert.Close()
+	if err != nil {
+		return err
 	}
+
+	_, err = insert.Exec(card.Id, card.Name, card.Set, card.Foil, card.NonFoil, card.Reserved, card.ContentWarning, card.Promo, card.CollectorNumber, card.Language)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }

sevenkeys/database/operations/selects.go

@@ -3,6 +3,7 @@ package operations
 import (
 	"database/sql"
 	"sevenkeys/database/entities"
+	"time"
 )
 
 func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) {
@@ -13,3 +14,21 @@ func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) {
 
 	return gamepiece, err
 }
+
+func GetCacheTimestampByType(db *sql.DB, cacheType string) (time.Time, error) {
+	var timestamp string
+
+	query := "SELECT Stamp FROM CacheTimestamp WHERE CacheType = ?;"
+	err := db.QueryRow(query, cacheType).Scan(&timestamp)
+
+	if err == sql.ErrNoRows {
+		return time.Unix(0, 0), nil
+	}
+
+	stamp, err := time.Parse("2006-01-02 15:04:05", timestamp)
+	if err != nil {
+		return time.Unix(0, 0), err
+	}
+
+	return stamp, err
+}

sevenkeys/main.go

@@ -2,109 +2,202 @@ package main
 
 import (
 	"database/sql"
-	"fmt"
+	"encoding/json"
+	"io"
+	"io/ioutil"
 	"log"
-
+	"net/http"
-	"github.com/go-sql-driver/mysql"
+	"os"
+	"sevenkeys/database"
+	"sevenkeys/database/entities"
+	"sevenkeys/database/operations"
+	"sevenkeys/scryfall/methods"
+	"sevenkeys/scryfall/types"
+	"time"
 )
 
-type Product struct {
+const CACHE_DIR string = "cache"
-	Name      string
-	Price     float64
-	Available bool
-}
 
-var db *sql.DB
+const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/"
+const SET_ICON_FILE_EXTENSION string = ".svg"
 
-func main() {
+const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json"
-	connectionConfig := mysql.Config{
-		User:                 "root",
-		Passwd:               "o7MS6CIn660jIApSP",
-		Net:                  "tcp",
-		Addr:                 "127.0.0.1:3306",
-		DBName:               "goql",
-		AllowNativePasswords: true,
-	}
 
-	var err error
+const GAME_PAPER = "paper"
 
-	db, err := sql.Open("mysql", connectionConfig.FormatDSN())
+func check(err error) {
-	defer db.Close()
-
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if err = db.Ping(); err != nil {
-		log.Fatal(err)
-	}
-	fmt.Println("Connected!")
-
-	createProductTable(db)
-
-	insertProduct(db, Product{Name: "Book 1", Price: 15.95, Available: true})
-	insertProduct(db, Product{Name: "Book 2", Price: 15.95, Available: true})
-	insertProduct(db, Product{Name: "Book 3", Price: 15.95, Available: true})
-	insertProduct(db, Product{Name: "Book 4", Price: 15.95, Available: true})
-	insertProduct(db, Product{Name: "Book 5", Price: 15.95, Available: true})
-
-	products := getAllProducts(db)
-
-	fmt.Println(products)
-}
-
-func createProductTable(db *sql.DB) {
-	query := `CREATE TABLE IF NOT EXISTS Product (
-		Id SERIAL PRIMARY KEY,
-		Name VARCHAR(100) NOT NULL,
-		Price NUMERIC(6,2) NOT NULL,
-		Available BOOLEAN,
-		Created TIMESTAMP DEFAULT NOW()
-	)`
-
-	_, err := db.Exec(query)
 	if err != nil {
 		log.Fatal(err)
 	}
 }
 
-func insertProduct(db *sql.DB, product Product) int64 {
+func createCacheDirectories() error {
-	query := "INSERT INTO Product (Name, Price, Available) VALUES (?, ?, ?);"
+	err := os.Mkdir(CACHE_DIR, os.ModePerm)
-
+	if err != nil && !os.IsExist(err) {
-	insert, err := db.Prepare(query)
+		return err
-	defer insert.Close()
-	if err != nil {
-		log.Fatal(err)
 	}
 
-	result, err := insert.Exec(product.Name, product.Price, product.Available)
+	err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm)
-	rowsAffected, _ := result.RowsAffected()
+	if err != nil && !os.IsExist(err) {
-	if err != nil || rowsAffected != 1 {
+		return err
-		log.Fatal(err)
 	}
 
-	insertedRowId, _ := result.LastInsertId()
+	return nil
-	return insertedRowId
 }
 
-func getAllProducts(db *sql.DB) []Product {
+func importSets(db *sql.DB, sets []types.Set) error {
-	products := []Product{}
+	for _, set := range sets {
-
+		// We're only interested in paper cards, so skip importing
-	rows, err := db.Query("SELECT Name, Price, Available FROM Product;")
+		// any sets that were only released in a video game
-	if err != nil {
+		if set.Digital {
-		log.Fatal(err)
+			continue
-	}
-	defer rows.Close()
-
-	var product Product
-	for rows.Next() {
-		err := rows.Scan(&product.Name, &product.Price, &product.Available)
-		if err != nil {
-			log.Fatal(err)
 		}
 
-		products = append(products, product)
+		log.Println("Importing " + set.Code)
+		err := operations.InsertOrUpdateSet(db, set)
+		if err != nil {
+			return err
+		}
+
+		log.Println("Downloading logo for " + set.Code)
+		response, err := http.Get(set.IconSvgUri)
+		defer response.Body.Close()
+		if err != nil {
+			return err
+		}
+
+		iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION
+		iconFile, err := os.Create(iconFilename)
+		if err != nil {
+			return err
+		}
+
+		io.Copy(iconFile, response.Body)
+		log.Println("Finished importing " + set.Code)
 	}
 
-	return products
+	return nil
+}
+
+func cardsUpdatedSinceCache(db *sql.DB, updatedAtTimestamp time.Time) (bool, error) {
+	cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData)
+	if err != nil {
+		return false, err
+	}
+
+	return updatedAtTimestamp.After(cachedFileTimestamp), nil
+}
+
+func cacheAllCardsFile(db *sql.DB, uri string, updatedAtTimestamp time.Time) error {
+	log.Printf("Downloading bulk card data...")
+	bulkCardsResponse, err := http.Get(uri)
+	if err != nil {
+		return err
+	}
+	log.Printf("Downloaded bulk card data.")
+
+	log.Printf("Writing card data to cache file...")
+	cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
+	if err != nil {
+		return err
+	}
+
+	defer bulkCardsResponse.Body.Close()
+	defer cacheFile.Close()
+	io.Copy(cacheFile, bulkCardsResponse.Body)
+	log.Printf("Cache file written.")
+
+	log.Printf("Recording timestamp...")
+	err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
+	if err != nil {
+		return err
+	}
+	log.Printf("Timestamp recorded.")
+
+	return nil
+}
+
+func importCards(db *sql.DB, cards []types.Card) error {
+	for _, card := range cards {
+		// We're only interested in paper cards, so skip cards or printings of a card which
+		// aren't available in paper
+		if !isPaper(card) {
+			continue
+		}
+
+		err := operations.InsertCard(db, card)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func isPaper(card types.Card) bool {
+	var paper bool = false
+
+	for _, game := range card.Games {
+		if game == GAME_PAPER {
+			paper = true
+		}
+	}
+
+	return paper
+}
+
+func main() {
+	log.Println("Connecting to database...")
+	db := database.GetDatabaseFromConfig("config.json")
+	log.Println("Connected.")
+
+	log.Println("Creating cache directories...")
+	err := createCacheDirectories()
+	check(err)
+	log.Println("Created cache directories.")
+
+	log.Println("Downloading set data from Scryfall...")
+	sets, err := methods.GetSets()
+	check(err)
+	log.Println("Downloaded set data.")
+
+	log.Println("Importing set data...")
+	err = importSets(db, sets)
+	check(err)
+	log.Println("Imported sets.")
+
+	log.Println("Downloading bulk data entry...")
+	allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards)
+	check(err)
+	log.Println("Downloaded bulk data entry.")
+
+	updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
+	check(err)
+	updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second) // Remove the fractional seconds component from the timestamp
+
+	log.Println("Checking for redownload...")
+	updated, err := cardsUpdatedSinceCache(db, updatedAtTimestamp)
+	check(err)
+
+	if updated {
+		log.Printf("Bulk data has been updated since last cache, redownloading.")
+		err = cacheAllCardsFile(db, allCardsBulkData.DownloadUri, updatedAtTimestamp)
+		check(err)
+	} else {
+		log.Printf("Bulk data has not been updated. Skipping download.")
+	}
+
+	log.Printf("Unmarsaling file into slice...")
+	allCardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME)
+	check(err)
+
+	var allCards []types.Card
+	err = json.Unmarshal(allCardsBytes, &allCards)
+	check(err)
+	log.Printf("Unmarshaled file.")
+
+	log.Printf("Importing card data into database...")
+	err = importCards(db, allCards)
+	check(err)
+	log.Printf("Imported card data.")
 }

sevenkeys/mtggoldfish/decksearch.go

@@ -1,98 +0,0 @@
-package mtggoldfish
-
-import (
-	"fmt"
-
-	"github.com/PuerkitoBio/goquery"
-	"github.com/geziyor/geziyor"
-	"github.com/geziyor/geziyor/client"
-	"github.com/geziyor/geziyor/export"
-)
-
-var DECK_SEARCH_URL string = "https://www.mtggoldfish.com/deck_searches/create"
-
-const (
-	TypeMaindeck  = "maindeck"
-	TypeSideboard = "sideboard"
-	TypeCommander = "commander"
-)
-
-type DeckSearchIncludedCard struct {
-	Name     string
-	Quantity int
-	Type     string
-}
-
-type DeckSearchParameters struct {
-	Name                   string
-	Format                 string
-	IncludeTournamentDecks bool
-	IncludeUserDecks       bool
-	Player                 string
-	StartDate              string // TODO: Date type?
-	EndDate                string // TODO: Date type?
-	IncludedCards          []DeckSearchCard
-}
-
-func (p *DeckSearchParameters) String() string {
-	searchUrl := DECK_SEARCH_URL + "?utf8=✓" +
-		"&deck_search[name]=" + p.Name +
-		"&deck_search[format]=" + p.Format +
-		"&deck_search[types][]="
-
-	if p.IncludeTournamentDecks {
-		searchUrl = searchUrl + "&deck_search[types][]=tournament"
-	}
-
-	if p.IncludeUserDecks {
-		searchUrl = searchUrl + "&deck_search[types][]=user"
-	}
-
-	searchUrl = searchUrl + "&deck_search[player]=" + p.Player
-	//"&deck_search[date_range]=" + p.StartDate + " - " + p.EndDate
-
-	return searchUrl
-}
-
-type DeckSearchCard struct {
-	Name     string
-	Quantity int
-	Type     string
-}
-
-type DeckSearchDecklist struct {
-	MtgGoldfishId int
-	Date          string // TODO: Date type?
-	Name          string
-	Source        string
-	Format        string
-	Author        string
-	Cards         []DeckSearchCard
-}
-
-type DeckSearchResults struct {
-	Decklists []DeckSearchDecklist
-	DeckCount int
-	PageCount int
-}
-
-func parseSearchResults(g *geziyor.Geziyor, r *client.Response) {
-	r.HTMLDoc.Find("div.table-responsive").Each(func(i int, s *goquery.Selection) {
-		g.Exports <- map[string]interface{}{
-			"test": s.Find("table>thead>tr>th").Text(),
-		}
-	})
-}
-
-func DeckSearch(params DeckSearchParameters) (DeckSearchResults, error) {
-	searchUrl := `https://www.mtggoldfish.com/deck_searches/create?utf8=✓&deck_search[name]=Burn&deck_search[format]=pauper&deck_search[types][]=&deck_search[types][]=tournament&deck_search[types][]=user&deck_search[player]=Jirach1&deck_search[date_range]=04%2F14%2F2024+-+04%2F28%2F2024&deck_search[deck_search_card_filters_attributes][0][card]=Kuldotha+Rebirth&deck_search[deck_search_card_filters_attributes][0][quantity]=4&deck_search[deck_search_card_filters_attributes][0][type]=maindeck&deck_search[deck_search_card_filters_attributes][1][card]=&deck_search[deck_search_card_filters_attributes][1][quantity]=1&deck_search[deck_search_card_filters_attributes][1][type]=maindeck&deck_search[deck_search_card_filters_attributes][2][card]=&deck_search[deck_search_card_filters_attributes][2][quantity]=1&deck_search[deck_search_card_filters_attributes][2][type]=maindeck&counter=3&commit=Search`
-	fmt.Println(searchUrl)
-
-	geziyor.NewGeziyor(&geziyor.Options{
-		StartURLs: []string{searchUrl},
-		ParseFunc: parseSearchResults,
-		Exporters: []export.Exporter{&export.JSON{}},
-	}).Start()
-
-	return DeckSearchResults{}, nil
-}

sevenkeys/scryfall/methods/bulkdata.go

@@ -4,76 +4,33 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
-	"log"
 	"net/http"
 	"sevenkeys/scryfall/types"
 )
 
 var BULK_DATA_URI = "https://api.scryfall.com/bulk-data"
 
-func GetBulkData() types.BulkDataList {
+func GetBulkDataByType(bulkDataType string) (types.BulkData, error) {
-	response, err := http.Get(BULK_DATA_URI)
+	response, err := http.Get(BULK_DATA_URI + "/" + bulkDataType)
 	if err != nil {
-		log.Fatal(err)
+		return types.BulkData{}, err
 	}
 
 	if response.StatusCode != http.StatusOK {
-		log.Fatal(response.StatusCode)
+		return types.BulkData{}, errors.New("HTTP request failed with code: " + string(response.StatusCode))
 	}
 
 	defer response.Body.Close()
 	bulkDataBytes, err := io.ReadAll(response.Body)
 	if err != nil {
-		log.Fatal(err)
+		return types.BulkData{}, err
 	}
 
-	var bulkDataList types.BulkDataList
+	var bulkData types.BulkData
-	err = json.Unmarshal(bulkDataBytes, &bulkDataList)
+	err = json.Unmarshal(bulkDataBytes, &bulkData)
 	if err != nil {
-		log.Fatal(err)
+		return types.BulkData{}, err
 	}
 
-	return bulkDataList
+	return bulkData, nil
-}
-
-func getBulkDownloadUri(bulkType string) (string, error) {
-	bulkDataList := GetBulkData()
-	for index := range bulkDataList.Data {
-		bulkData := bulkDataList.Data[index]
-		if bulkData.Type == bulkType {
-			return bulkData.DownloadUri, nil
-		}
-	}
-
-	return "", errors.New("No bulk data of type " + bulkType + " found.")
-}
-
-func GetAllCards() []types.Card {
-	allCardsUri, err := getBulkDownloadUri("all_cards")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	response, err := http.Get(allCardsUri)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if response.StatusCode != http.StatusOK {
-		log.Fatal(response.StatusCode)
-	}
-
-	defer response.Body.Close()
-	allCardsBytes, err := io.ReadAll(response.Body)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	var allCards []types.Card
-	err = json.Unmarshal(allCardsBytes, &allCards)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return allCards
 }

sevenkeys/scryfall/methods/sets.go

@@ -1,3 +1,36 @@
 package methods
 
-func GetSets() {}
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"sevenkeys/scryfall/types"
+)
+
+const SETS_API_URL string = "https://api.scryfall.com/sets"
+
+func GetSets() ([]types.Set, error) {
+	response, err := http.Get(SETS_API_URL)
+	if err != nil {
+		return []types.Set{}, nil
+	}
+
+	if response.StatusCode != http.StatusOK {
+		return []types.Set{}, errors.New("HTTP request failed with code: " + string(response.StatusCode))
+	}
+
+	defer response.Body.Close()
+	setsBytes, err := io.ReadAll(response.Body)
+	if err != nil {
+		return []types.Set{}, err
+	}
+
+	var setList types.SetList
+	err = json.Unmarshal(setsBytes, &setList)
+	if err != nil {
+		return []types.Set{}, err
+	}
+
+	return setList.Data, nil
+}

sevenkeys/scryfall/types/bulkdata.go

@@ -1,5 +1,11 @@
 package types
 
+const BulkDataTypeOracleCards string = "oracle_cards"
+const BulkDataTypeUniqueArtwork string = "unique_artwork"
+const BulkDataTypeDefaultCards string = "default_cards"
+const BulkDataTypeAllCards string = "all_cards"
+const BulkDataTypeRulings string = "rulings"
+
 type BulkData struct {
 	Id              string `json:"id"`
 	Uri             string `json:"uri"`
@@ -12,9 +18,3 @@ type BulkData struct {
 	ContentType     string `json:"content_type"`
 	ContentEncoding string `json:"content_encoding"`
 }
-
-type BulkDataList struct {
-	Object  string     `json:"object"`
-	HasMore bool       `json:"has_more"`
-	Data    []BulkData `json:"data"`
-}

sevenkeys/scryfall/types/card.go

@@ -0,0 +1,15 @@
+package types
+
+type Card struct {
+	Id              string   `json:"id"` // GUID
+	Name            string   `json:"name"`
+	Set             string   `json:"set"`
+	Games           []string `json:"games"`
+	Foil            bool     `json:"foil"`
+	NonFoil         bool     `json:"nonfoil"`
+	Reserved        bool     `json:"reserved"`
+	ContentWarning  bool     `json:"content_warning,omitempty"`
+	Promo           bool     `json:"promo"`
+	CollectorNumber string   `json:"collector_number"`
+	Language        string   `json:"lang"`
+}

sevenkeys/scryfall/types/cards.go

@@ -1,130 +0,0 @@
-package types
-
-type Colors []string
-
-type RelatedCard struct {
-	Id        string `json:"id"`
-	Object    string `json:"object"`
-	Component string `json:"component"`
-	Name      string `json:"name"`
-	TypeLine  string `json:"type_line"`
-	ApiUri    string `json:"uri"`
-}
-
-type CardFace struct {
-	Artist          string            `json:"artist"`
-	ArtistId        string            `json:"artist_id"`
-	ManaValue       float32           `json:"cmc"`
-	ColorIndicator  Colors            `json:"color_indicator"`
-	Colors          Colors            `json:"colors"`
-	Defense         string            `json:"defense"`
-	FlavorText      string            `json:"flavor_text"`
-	IllustrationId  string            `json:"illustration_id"`
-	ImageUris       map[string]string `json:"image_uris"` // TODO: Find out the structure of this object
-	Layout          string            `json:"layout"`
-	Loyalty         string            `json:"loyalty"`
-	ManaCost        string            `json:"mana_cost"`
-	Name            string            `json:"name"`
-	Object          string            `json:"object"`
-	OracleId        string            `json:"oracle_id"`
-	Power           string            `json:"power"`
-	PrintedName     string            `json:"printed_name"`
-	PrintedText     string            `json:"printed_text"`
-	PrintedTypeLine string            `json:"printed_type_line"`
-	Toughness       string            `json:"toughness"`
-	TypeLine        string            `json:"type_line"`
-	Watermark       string            `json:"watermark"`
-}
-
-type Card struct {
-	// Core fields
-	ArenaId           int    `json:"arena_id"`
-	Id                string `json:"id"`
-	Language          string `json:"lang"`
-	MtgoId            int    `json:"mtgo_id"`
-	MtgoFoilid        int    `json:"mtgo_foil_id"`
-	MultiverseIds     []int  `json:"multiverse_ids"`
-	TcgplayerId       int    `json:"tcgplayer_id"`
-	TcgplayerEtchedId int    `json:"tcgplayer_etched_id"`
-	Object            string `json:"object"`
-	Layout            string `json:"layout"`
-	OracleId          string `json:"oracle_id"`
-	PrintsSearchUri   string `json:"prints_search_uri"`
-	RulingsUri        string `json:"rulings_uri"`
-	ScryfallUri       string `json:"scryfall_uri"`
-	ApiUri            string `json:"uri"`
-
-	// Gameplay fields
-	AllParts          []RelatedCard     `json:"all_parts"`
-	CardFaces         []CardFace        `json:"card_faces"`
-	ManaValue         float32           `json:"cmc"`
-	ColorIdentity     Colors            `json:"color_identity"`
-	ColorIndicator    Colors            `json:"color_indicator"`
-	Colors            Colors            `json:"colors"`
-	Defense           string            `json:"defense"`
-	EdhrecRank        int               `json:"edhrec_rank"`
-	HandModifier      string            `json:"hand_modifier"`
-	Keywords          []string          `json:"keywords"`
-	Legalities        map[string]string `json:"legalities"`
-	LifeModifier      string            `json:"life_modifier"`
-	Loyalty           string            `json:"loyalty"`
-	ManaCost          string            `json:"mana_cost"`
-	Name              string            `json:"name"`
-	OracleText        string            `json:"oracle_text"`
-	PennyDreadfulRank int               `json:"penny_rank"`
-	Power             string            `json:"power"`
-	ProducedMana      Colors            `json:"produced_mana"`
-	ReserveList       bool              `json:"reserved"`
-	Toughness         string            `json:"toughness"`
-	TypeLine          string            `json:"type_line"`
-
-	// Printing fields
-	Artist           string            `json:"artist"`
-	ArtistIds        []string          `json:"artist_ids"`
-	AttractionLights []interface{}     `json:"attraction_lights"` // TODO: Figure out schema
-	Booster          bool              `json:"booster"`
-	BorderColor      string            `json:"border_color"`
-	CardBackId       string            `json:"card_back_id"`
-	CollectorNumber  string            `json:"collector_number"`
-	ContentWarning   bool              `json:"content_warning"`
-	Digital          bool              `json:"digital"`
-	Finishes         []interface{}     `json:"finishes"` // TODO: Find out how flags are formatted
-	FlavorName       string            `json:"flavor_name"`
-	FlavorText       string            `json:"flavor_text"`
-	FrameEffects     []string          `json:"frame_effects"`
-	Frame            string            `json:"frame"`
-	FullArt          bool              `json:"full_art"`
-	Games            []string          `json:"games"`
-	HighresImage     bool              `json:"highres_image"`
-	IllustrationId   string            `json:"illustration_id"`
-	ImageStatus      string            `json:"image_status"`
-	ImageUris        map[string]string `json:"image_uris"` // TODO: Find out shape of object
-	Oversized        bool              `json:"oversized"`
-	Prices           map[string]string `json:"prices"`
-	PrintedName      string            `json:"printed_name"`
-	PrintedText      string            `json:"printed_text"`
-	PrintedTypeLine  string            `json:"printed_type_line"`
-	Promo            bool              `json:"promo"`
-	PromoTypes       []string          `json:"promo_types"`
-	PurchaseUris     interface{}       `json:"purchase_uris"` // TODO: Find out shape of object
-	Rarity           string            `json:"rarity"`
-	RelatedUris      interface{}       `json:"related_uris"` // TODO: Find out shape of object
-	ReleasedAt       string            `json:"released_at"`  // TODO: Datetime type?
-	Reprint          bool              `json:"reprint"`
-	ScryfallSetUri   string            `json:"scryfall_set_uri"`
-	SetName          string            `json:"set_name"`
-	SetSearchUri     string            `json:"set_search_uri"`
-	SetType          string            `json:"set_type"`
-	SetUri           string            `json:"set_uri"`
-	Set              string            `json:"set"`
-	SetId            string            `json:"set_id"`
-	StorySpotlight   bool              `json:"story_spotlight"`
-	Textless         bool              `json:"textless"`
-	Variation        bool              `json:"variation"`
-	VariationOf      string            `json:"variation_of"`
-	SecurityStamp    string            `json:"security_stamp"`
-	Watermark        string            `json:"watermark"`
-	PreviewedAt      string            `json:"preview.previewed_at"`
-	PreviewSourceUri string            `json:"preview.source_uri"`
-	PreviewSource    string            `json:"preview.source"`
-}

sevenkeys/scryfall/types/set.go

@@ -0,0 +1,15 @@
+package types
+
+type SetList struct {
+	Object  string `json:"object"`
+	HasMore bool   `json:"has_more"`
+	Data    []Set  `json:"data"`
+}
+
+type Set struct {
+	Code       string `json:"code"`
+	Name       string `json:"name"`
+	CardCount  int    `json:"card_count"`
+	IconSvgUri string `json:"icon_svg_uri"`
+	Digital    bool   `json:"digital"`
+}

sevenkeys/scryfall/types/timestamp.go

@@ -0,0 +1,3 @@
+package types
+
+const ScryfallTimestampFormat = "2006-01-02T15:04:05.999-07:00"

sevenkeys/sql/createdb.sql

@@ -2,38 +2,28 @@ CREATE DATABASE IF NOT EXISTS sevenkeys;
 
 USE sevenkeys;
 
-CREATE TABLE IF NOT EXISTS Gamepiece (
+CREATE TABLE IF NOT EXISTS CacheTimestamp (
-	Id INT AUTO_INCREMENT PRIMARY KEY,
+	CacheType ENUM('AllCardsBulkData') PRIMARY KEY,
-	Name VARCHAR(150) NOT NULL
+	Stamp DATETIME NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS ExpansionSet (
+	SetCode VARCHAR(6) PRIMARY KEY,
+	Name VARCHAR(60) NOT NULL,
+	CardCount INT NOT NULL,
+	IconSvgUri VARCHAR(60) NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS CardPrinting (
-	Id INT AUTO_INCREMENT PRIMARY KEY,
+	Id VARCHAR(36) PRIMARY KEY, -- GUID
-	GamepieceId INT NOT NULL,
+	Name VARCHAR(150) NOT NULL,
-	FOREIGN KEY (GamepieceId) REFERENCES Gamepiece(Id),
 	SetCode VARCHAR(6) NOT NULL,
-	ImageUrl VARCHAR(2048) NOT NULL
+	FOREIGN KEY (SetCode) REFERENCES ExpansionSet(SetCode),
-);
+	HasFoil BOOLEAN NOT NULL,
-
+	HasNonFoil BOOLEAN NOT NULL,
-CREATE TABLE IF NOT EXISTS TournamentDecklist (
+	IsReserved BOOLEAN NOT NULL,
-	Id INT AUTO_INCREMENT PRIMARY KEY,
+	IsRacist BOOLEAN NOT NULL,
-	DeckName VARCHAR(100) NOT NULL,
+	IsPromo BOOLEAN NOT NULL,
-	DatePublished DATE NOT NULL,
+	CollectorNumber VARCHAR(10) NOT NULL,
-	Source VARCHAR(200) NOT NULL,
+	Language VARCHAR(3) NOT NULL
-	Format VARCHAR(25) NOT NULL,
-	AuthorName VARCHAR(100) NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS TournamentDecklistCard (
-	TournamentDecklistId INT NOT NULL,
-	GamepieceId INT NOT NULL,
-	PRIMARY KEY (TournamentDecklistId, GamepieceId),
-
-	FOREIGN KEY TournamentDecklistId REFERENCES TournamentDecklist(Id),
-	FOREIGN KEY GamepieceId REFERENCES Gamepiece(Id),
-
-	CardPrintingId INT NULL,
-	FOREIGN KEY CardPrintingId REFERENCES CardPrinting(Id),
-
-	Quantity INT NOT NULL
 );

sevenkeys/test.json

@@ -0,0 +1 @@
+{"object":"card","id":"0000579f-7b35-4ed3-b44c-db2a538066fe","oracle_id":"44623693-51d6-49ad-8cd7-140505caf02f","multiverse_ids":[109722],"mtgo_id":25527,"mtgo_foil_id":25528,"tcgplayer_id":14240,"cardmarket_id":13850,"name":"Fury Sliver","lang":"en","released_at":"2006-10-06","uri":"https://api.scryfall.com/cards/0000579f-7b35-4ed3-b44c-db2a538066fe","scryfall_uri":"https://scryfall.com/card/tsp/157/fury-sliver?utm_source=api","layout":"normal","highres_image":true,"image_status":"highres_scan","image_uris":{"small":"https://cards.scryfall.io/small/front/0/0/0000579f-7b35-4ed3-b44c-db2a538066fe.jpg?1562894979","normal":"https://cards.scryfall.io/normal/front/0/0/0000579f-7b35-4ed3-b44c-db2a538066fe.jpg?1562894979","large":"https://cards.scryfall.io/large/front/0/0/0000579f-7b35-4ed3-b44c-db2a538066fe.jpg?1562894979","png":"https://cards.scryfall.io/png/front/0/0/0000579f-7b35-4ed3-b44c-db2a538066fe.png?1562894979","art_crop":"https://cards.scryfall.io/art_crop/front/0/0/0000579f-7b35-4ed3-b44c-db2a538066fe.jpg?1562894979","border_crop":"https://cards.scryfall.io/border_crop/front/0/0/0000579f-7b35-4ed3-b44c-db2a538066fe.jpg?1562894979"},"mana_cost":"{5}{R}","cmc":6.0,"type_line":"Creature — Sliver","oracle_text":"All Sliver creatures have double strike.","power":"3","toughness":"3","colors":["R"],"color_identity":["R"],"keywords":[],"legalities":{"standard":"not_legal","future":"not_legal","historic":"not_legal","timeless":"not_legal","gladiator":"not_legal","pioneer":"not_legal","explorer":"not_legal","modern":"legal","legacy":"legal","pauper":"not_legal","vintage":"legal","penny":"not_legal","commander":"legal","oathbreaker":"legal","standardbrawl":"not_legal","brawl":"not_legal","alchemy":"not_legal","paupercommander":"restricted","duel":"legal","oldschool":"not_legal","premodern":"not_legal","predh":"legal"},"games":["paper","mtgo"],"reserved":false,"foil":true,"nonfoil":true,"finishes":["nonfoil","foil"],"oversized":false,"promo":false,"reprint":false,"variation":false,"set_id":"c1d109bc-ffd8-428f-8d7d-3f8d7e648046","set":"tsp","set_name":"Time Spiral","set_type":"expansion","set_uri":"https://api.scryfall.com/sets/c1d109bc-ffd8-428f-8d7d-3f8d7e648046","set_search_uri":"https://api.scryfall.com/cards/search?order=set&q=e%3Atsp&unique=prints","scryfall_set_uri":"https://scryfall.com/sets/tsp?utm_source=api","rulings_uri":"https://api.scryfall.com/cards/0000579f-7b35-4ed3-b44c-db2a538066fe/rulings","prints_search_uri":"https://api.scryfall.com/cards/search?order=released&q=oracleid%3A44623693-51d6-49ad-8cd7-140505caf02f&unique=prints","collector_number":"157","digital":false,"rarity":"uncommon","flavor_text":"\"A rift opened, and our arrows were abruptly stilled. To move was to push the world. But the sliver's claw still twitched, red wounds appeared in Thed's chest, and ribbons of blood hung in the air.\"\n—Adom Capashen, Benalish hero","card_back_id":"0aeebaf5-8c7d-4636-9e82-8c27447861f7","artist":"Paolo Parente","artist_ids":["d48dd097-720d-476a-8722-6a02854ae28b"],"illustration_id":"2fcca987-364c-4738-a75b-099d8a26d614","border_color":"black","frame":"2003","full_art":false,"textless":false,"booster":true,"story_spotlight":false,"edhrec_rank":7361,"penny_rank":10143,"prices":{"usd":"0.40","usd_foil":"1.80","usd_etched":null,"eur":"0.19","eur_foil":"1.02","tix":"0.03"},"related_uris":{"gatherer":"https://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid=109722&printed=false","tcgplayer_infinite_articles":"https://tcgplayer.pxf.io/c/4931599/1830156/21018?subId1=api&trafcat=infinite&u=https%3A%2F%2Finfinite.tcgplayer.com%2Fsearch%3FcontentMode%3Darticle%26game%3Dmagic%26partner%3Dscryfall%26q%3DFury%2BSliver","tcgplayer_infinite_decks":"https://tcgplayer.pxf.io/c/4931599/1830156/21018?subId1=api&trafcat=infinite&u=https%3A%2F%2Finfinite.tcgplayer.com%2Fsearch%3FcontentMode%3Ddeck%26game%3Dmagic%26partner%3Dscryfall%26q%3DFury%2BSliver","edhrec":"https://edhrec.com/route/?cc=Fury+Sliver"},"purchase_uris":{"tcgplayer":"https://tcgplayer.pxf.io/c/4931599/1830156/21018?subId1=api&u=https%3A%2F%2Fwww.tcgplayer.com%2Fproduct%2F14240%3Fpage%3D1","cardmarket":"https://www.cardmarket.com/en/Magic/Products/Singles/Time-Spiral/Fury-Sliver?referrer=scryfall&utm_campaign=card_prices&utm_medium=text&utm_source=scryfall","cardhoarder":"https://www.cardhoarder.com/cards/25527?affiliate_id=scryfall&ref=card-profile&utm_campaign=affiliate&utm_medium=card&utm_source=scryfall"}},