Compare commits

...

36 Commits

Author SHA1 Message Date
The Magician 411abd66f6 Skip digital-only cards, close insert statements 2024-05-21 15:52:26 +01:00
The Magician d21f1274a6 Remove debug code 2024-05-21 15:27:36 +01:00
The Magician a7b8b66db3 Add card insert code 2024-05-21 15:27:05 +01:00
The Magician af6a9d0feb Minor fixen 2024-05-21 15:17:25 +01:00
The Magician bd581e1c60 Add correct foil flags 2024-05-21 15:17:00 +01:00
The Magician 07e8277a00 Remove importcards command 2024-05-21 15:16:43 +01:00
The Magician 9fa8bb563f Unmarshal card data into slice 2024-05-21 15:15:58 +01:00
The Magician 4c8beb0eb5 Add Card Scryfall type with only required fields 2024-05-21 14:41:35 +01:00
The Magician d1fa838bfe Cache bulk data file 2024-05-21 14:31:54 +01:00
The Magician 01e06c0152 Fix timestamp SELECT code 2024-05-21 14:31:20 +01:00
The Magician 411383704c Import set data 2024-05-21 13:25:08 +01:00
The Magician fd4c2048e1 Modify Makefile to use unified importer 2024-05-21 12:35:01 +01:00
The Magician 787794ac97 Move set import code to main.go 2024-05-21 12:33:14 +01:00
The Magician a01e591e5f Create cache directories 2024-05-21 12:28:08 +01:00
The Magician f31186add2 Write outline for combined importer 2024-05-21 12:24:16 +01:00
The Magician e98b0ec72f Move filesystem constants to main.go 2024-05-21 12:19:01 +01:00
The Magician a6463b0b25 Implement check function 2024-05-21 12:17:50 +01:00
The Magician b25cab0104 Remove test code from main.go 2024-05-21 12:17:24 +01:00
The Magician 5f7f07edea Move loop to main package 2024-05-20 17:33:40 +01:00
The Magician c4f87dc451 Change Makefile targets 2024-05-20 17:26:18 +01:00
The Magician d3583cee0e Make "CacheTimestamp" table name singular 2024-05-20 17:21:16 +01:00
The Magician d74d299418 Skip importing digital-only sets 2024-05-20 17:20:27 +01:00
The Magician 75b425c849 Import sets into database 2024-05-20 17:17:45 +01:00
The Magician 3f9d3a9dd6 Add missing comma 2024-05-20 16:54:49 +01:00
The Magician 2131a54e64 Remove unused struct 2024-05-20 16:50:57 +01:00
The Magician 72452ee2fa Start working on speeding up imports 2024-05-19 20:29:50 +01:00
The Magician af27b138bc Finish bulk json caching feature 2024-05-19 17:11:42 +01:00
The Magician e5b87bb8ae Remove unused API method 2024-05-19 15:01:52 +01:00
The Magician c8133803f7 Cache bulk data file 2024-05-19 15:01:18 +01:00
The Magician 6def107873 Remove broken decklist functionality 2024-05-19 10:06:28 +01:00
The Magician 87a94f800d Fix SQL syntax 2024-05-19 10:05:29 +01:00
The Magician e89495ba80 Add table to store cache timestamps 2024-05-19 10:01:57 +01:00
The Magician 54d82711eb Convert to grayscale before doing edge detection 2024-05-15 12:19:06 +01:00
The Magician 5f5eae7443 Add sliders to adjust Canny thresholds 2024-05-15 11:16:50 +01:00
The Magician beed5fd92d Ignore build files 2024-05-14 15:50:06 +01:00
The Magician ec06a3aa22 Start A Scanner Darkly 2024-05-14 15:48:47 +01:00
21 changed files with 399 additions and 471 deletions

5
a_scanner_darkly/.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
cmake_install.cmake
CMakeCache.txt
CMakeFiles/*
Makefile
AScannerDarkly

View File

@ -0,0 +1,6 @@
cmake_minimum_required(VERSION 3.2)
project( AScannerDarkly )
find_package( OpenCV REQUIRED )
include_directories( ${OpenCV_INCLUDE_DIRS} )
add_executable( AScannerDarkly main.cpp )
target_link_libraries( AScannerDarkly ${OpenCV_LIBS} )

39
a_scanner_darkly/main.cpp Normal file
View File

@ -0,0 +1,39 @@
#include <opencv2/opencv.hpp>
const std::string WINDOW_NAME = "A Scanner Darkly";
const std::string LOWER_THRESHOLD_TRACKBAR_NAME = "Canny: Lower Threshold";
const std::string UPPER_THRESHOLD_TRACKBAR_NAME = "Canny: Upper Threshold";
int g_Canny_lower_threshold = 110;
int g_Canny_upper_threshold = 300;
int main(int argc, char** argv ) {
cv::VideoCapture cap;
cap.open(0);
if (!cap.isOpened()) {
std::cerr << "Couldn't open capture" << std::endl;
return -1;
}
cv::namedWindow(WINDOW_NAME);
cv::createTrackbar(LOWER_THRESHOLD_TRACKBAR_NAME, WINDOW_NAME, &g_Canny_lower_threshold, 1000, NULL);
cv::createTrackbar(UPPER_THRESHOLD_TRACKBAR_NAME, WINDOW_NAME, &g_Canny_upper_threshold, 1000, NULL);
cv::Mat frame, grayscaleFrame, cannyFrame;
while (true) {
cap >> frame;
cv::cvtColor(frame, grayscaleFrame, cv::COLOR_BGR2GRAY);
cv::Canny(grayscaleFrame, cannyFrame, g_Canny_lower_threshold, g_Canny_upper_threshold);
cv::imshow(WINDOW_NAME, cannyFrame);
char c = (char)cv::waitKey(33);
if (c == 27) {
break;
}
}
cv::destroyWindow(WINDOW_NAME);
}

View File

@ -1 +1,2 @@
cache/
cmd/database/config.json

View File

@ -1,10 +1,8 @@
removedb:
mysql --user=root --password=$(shell pass show sevenkeys/mysql) <sql/removedb.sql
createdb:
mysql --user=root --password=$(shell pass show sevenkeys/mysql) <sql/createdb.sql
importcards: createdb
go run cmd/importcards/main.go
import: createdb
go run main.go
getdecks:
go run cmd/getdecks/main.go
clean:
mysql --user=root --password=$(shell pass show sevenkeys/mysql) <sql/removedb.sql
rm -rf cache/

View File

@ -1,18 +0,0 @@
package main
import (
"fmt"
"sevenkeys/mtggoldfish"
)
func main() {
params := mtggoldfish.DeckSearchParameters{
Format: "pauper",
IncludeTournamentDecks: true,
StartDate: "04/14/2024",
EndDate: "04/28/2024",
}
decks, _ := mtggoldfish.DeckSearch(params)
fmt.Println(decks)
}

View File

@ -1,13 +0,0 @@
package main
import (
"sevenkeys/database"
"sevenkeys/database/operations"
"sevenkeys/scryfall/methods"
)
func main() {
db := database.GetDatabaseFromConfig("config.json")
allCards := methods.GetAllCards()
operations.InsertCards(db, allCards)
}

View File

@ -0,0 +1,3 @@
package entities
const CacheTypeAllCardsBulkData = "AllCardsBulkData"

View File

@ -2,57 +2,66 @@ package operations
import (
"database/sql"
"errors"
"log"
"sevenkeys/scryfall/types"
"time"
)
var ErrGamepieceExists error = errors.New("Gamepiece already exists in database")
func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error {
query := `INSERT INTO CacheTimestamp (CacheType, Stamp)
VALUES (?, ?)
ON DUPLICATE KEY
UPDATE Stamp = ?;`
func insertGamepiece(db *sql.DB, cardName string) (int, error) {
gamepieceInsertQuery := "INSERT INTO Gamepiece (Name) VALUES (?);"
gamepieceInsert, err := db.Prepare(gamepieceInsertQuery)
defer gamepieceInsert.Close()
insertOrUpdate, err := db.Prepare(query)
defer insertOrUpdate.Close()
if err != nil {
log.Fatal(err)
return err
}
existingGamepiece, err := GetGamepieceByName(db, cardName)
if err != sql.ErrNoRows {
return existingGamepiece.Id, ErrGamepieceExists
}
result, err := gamepieceInsert.Exec(cardName)
_, err = insertOrUpdate.Exec(cacheType, stamp, stamp)
if err != nil {
log.Fatal(err)
return err
}
gamepieceId, err := result.LastInsertId()
return nil
}
// TODO: There's really no need for this to be an "upsert", we can just ignore sets that we already have
func InsertOrUpdateSet(db *sql.DB, set types.Set) error {
query := `INSERT INTO ExpansionSet (SetCode, Name, CardCount, IconSvgUri)
VALUES (?, ?, ?, ?)
ON DUPLICATE KEY
Update Name = ?, CardCount = ?, IconSvgUri = ?;`
insertOrUpdate, err := db.Prepare(query)
defer insertOrUpdate.Close()
if err != nil {
log.Fatal(err)
return err
}
return int(gamepieceId), nil
}
func insertCardPrinting(db *sql.DB, gamepieceId int, setCode string, imageUrl string) {
cardPrintingInsertQuery := "INSERT INTO CardPrinting (GamepieceId, SetCode, ImageUrl) VALUES (?, ?, ?);"
cardPrintingInsert, err := db.Prepare(cardPrintingInsertQuery)
defer cardPrintingInsert.Close()
_, err = insertOrUpdate.Exec(set.Code, set.Name, set.CardCount, set.IconSvgUri, set.Name, set.CardCount, set.IconSvgUri)
if err != nil {
log.Fatal(err)
return err
}
_, err = cardPrintingInsert.Exec(gamepieceId, setCode, imageUrl)
return nil
}
func InsertCard(db *sql.DB, card types.Card) error {
query := `INSERT IGNORE INTO CardPrinting
(Id, Name, SetCode, HasFoil, HasNonFoil, IsReserved, IsRacist, IsPromo, CollectorNumber, Language)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`
insert, err := db.Prepare(query)
defer insert.Close()
if err != nil {
log.Fatal(err)
}
return err
}
func InsertCards(db *sql.DB, cards []types.Card) {
for index := range cards {
card := cards[index]
gamepieceId, _ := insertGamepiece(db, card.Name)
insertCardPrinting(db, gamepieceId, card.Set, card.ImageUris["png"])
_, err = insert.Exec(card.Id, card.Name, card.Set, card.Foil, card.NonFoil, card.Reserved, card.ContentWarning, card.Promo, card.CollectorNumber, card.Language)
if err != nil {
return err
}
return nil
}

View File

@ -3,6 +3,7 @@ package operations
import (
"database/sql"
"sevenkeys/database/entities"
"time"
)
func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) {
@ -13,3 +14,21 @@ func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) {
return gamepiece, err
}
func GetCacheTimestampByType(db *sql.DB, cacheType string) (time.Time, error) {
var timestamp string
query := "SELECT Stamp FROM CacheTimestamp WHERE CacheType = ?;"
err := db.QueryRow(query, cacheType).Scan(&timestamp)
if err == sql.ErrNoRows {
return time.Unix(0, 0), nil
}
stamp, err := time.Parse("2006-01-02 15:04:05", timestamp)
if err != nil {
return time.Unix(0, 0), err
}
return stamp, err
}

View File

@ -2,109 +2,202 @@ package main
import (
"database/sql"
"fmt"
"encoding/json"
"io"
"io/ioutil"
"log"
"github.com/go-sql-driver/mysql"
"net/http"
"os"
"sevenkeys/database"
"sevenkeys/database/entities"
"sevenkeys/database/operations"
"sevenkeys/scryfall/methods"
"sevenkeys/scryfall/types"
"time"
)
type Product struct {
Name string
Price float64
Available bool
const CACHE_DIR string = "cache"
const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/"
const SET_ICON_FILE_EXTENSION string = ".svg"
const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json"
const GAME_PAPER = "paper"
func check(err error) {
if err != nil {
log.Fatal(err)
}
}
var db *sql.DB
func createCacheDirectories() error {
err := os.Mkdir(CACHE_DIR, os.ModePerm)
if err != nil && !os.IsExist(err) {
return err
}
err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm)
if err != nil && !os.IsExist(err) {
return err
}
return nil
}
func importSets(db *sql.DB, sets []types.Set) error {
for _, set := range sets {
// We're only interested in paper cards, so skip importing
// any sets that were only released in a video game
if set.Digital {
continue
}
log.Println("Importing " + set.Code)
err := operations.InsertOrUpdateSet(db, set)
if err != nil {
return err
}
log.Println("Downloading logo for " + set.Code)
response, err := http.Get(set.IconSvgUri)
defer response.Body.Close()
if err != nil {
return err
}
iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION
iconFile, err := os.Create(iconFilename)
if err != nil {
return err
}
io.Copy(iconFile, response.Body)
log.Println("Finished importing " + set.Code)
}
return nil
}
func cardsUpdatedSinceCache(db *sql.DB, updatedAtTimestamp time.Time) (bool, error) {
cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData)
if err != nil {
return false, err
}
return updatedAtTimestamp.After(cachedFileTimestamp), nil
}
func cacheAllCardsFile(db *sql.DB, uri string, updatedAtTimestamp time.Time) error {
log.Printf("Downloading bulk card data...")
bulkCardsResponse, err := http.Get(uri)
if err != nil {
return err
}
log.Printf("Downloaded bulk card data.")
log.Printf("Writing card data to cache file...")
cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME)
if err != nil {
return err
}
defer bulkCardsResponse.Body.Close()
defer cacheFile.Close()
io.Copy(cacheFile, bulkCardsResponse.Body)
log.Printf("Cache file written.")
log.Printf("Recording timestamp...")
err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
if err != nil {
return err
}
log.Printf("Timestamp recorded.")
return nil
}
func importCards(db *sql.DB, cards []types.Card) error {
for _, card := range cards {
// We're only interested in paper cards, so skip cards or printings of a card which
// aren't available in paper
if !isPaper(card) {
continue
}
err := operations.InsertCard(db, card)
if err != nil {
return err
}
}
return nil
}
func isPaper(card types.Card) bool {
var paper bool = false
for _, game := range card.Games {
if game == GAME_PAPER {
paper = true
}
}
return paper
}
func main() {
connectionConfig := mysql.Config{
User: "root",
Passwd: "o7MS6CIn660jIApSP",
Net: "tcp",
Addr: "127.0.0.1:3306",
DBName: "goql",
AllowNativePasswords: true,
log.Println("Connecting to database...")
db := database.GetDatabaseFromConfig("config.json")
log.Println("Connected.")
log.Println("Creating cache directories...")
err := createCacheDirectories()
check(err)
log.Println("Created cache directories.")
log.Println("Downloading set data from Scryfall...")
sets, err := methods.GetSets()
check(err)
log.Println("Downloaded set data.")
log.Println("Importing set data...")
err = importSets(db, sets)
check(err)
log.Println("Imported sets.")
log.Println("Downloading bulk data entry...")
allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards)
check(err)
log.Println("Downloaded bulk data entry.")
updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
check(err)
updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second) // Remove the fractional seconds component from the timestamp
log.Println("Checking for redownload...")
updated, err := cardsUpdatedSinceCache(db, updatedAtTimestamp)
check(err)
if updated {
log.Printf("Bulk data has been updated since last cache, redownloading.")
err = cacheAllCardsFile(db, allCardsBulkData.DownloadUri, updatedAtTimestamp)
check(err)
} else {
log.Printf("Bulk data has not been updated. Skipping download.")
}
var err error
log.Printf("Unmarsaling file into slice...")
allCardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME)
check(err)
db, err := sql.Open("mysql", connectionConfig.FormatDSN())
defer db.Close()
var allCards []types.Card
err = json.Unmarshal(allCardsBytes, &allCards)
check(err)
log.Printf("Unmarshaled file.")
if err != nil {
log.Fatal(err)
}
if err = db.Ping(); err != nil {
log.Fatal(err)
}
fmt.Println("Connected!")
createProductTable(db)
insertProduct(db, Product{Name: "Book 1", Price: 15.95, Available: true})
insertProduct(db, Product{Name: "Book 2", Price: 15.95, Available: true})
insertProduct(db, Product{Name: "Book 3", Price: 15.95, Available: true})
insertProduct(db, Product{Name: "Book 4", Price: 15.95, Available: true})
insertProduct(db, Product{Name: "Book 5", Price: 15.95, Available: true})
products := getAllProducts(db)
fmt.Println(products)
}
func createProductTable(db *sql.DB) {
query := `CREATE TABLE IF NOT EXISTS Product (
Id SERIAL PRIMARY KEY,
Name VARCHAR(100) NOT NULL,
Price NUMERIC(6,2) NOT NULL,
Available BOOLEAN,
Created TIMESTAMP DEFAULT NOW()
)`
_, err := db.Exec(query)
if err != nil {
log.Fatal(err)
}
}
func insertProduct(db *sql.DB, product Product) int64 {
query := "INSERT INTO Product (Name, Price, Available) VALUES (?, ?, ?);"
insert, err := db.Prepare(query)
defer insert.Close()
if err != nil {
log.Fatal(err)
}
result, err := insert.Exec(product.Name, product.Price, product.Available)
rowsAffected, _ := result.RowsAffected()
if err != nil || rowsAffected != 1 {
log.Fatal(err)
}
insertedRowId, _ := result.LastInsertId()
return insertedRowId
}
func getAllProducts(db *sql.DB) []Product {
products := []Product{}
rows, err := db.Query("SELECT Name, Price, Available FROM Product;")
if err != nil {
log.Fatal(err)
}
defer rows.Close()
var product Product
for rows.Next() {
err := rows.Scan(&product.Name, &product.Price, &product.Available)
if err != nil {
log.Fatal(err)
}
products = append(products, product)
}
return products
log.Printf("Importing card data into database...")
err = importCards(db, allCards)
check(err)
log.Printf("Imported card data.")
}

View File

@ -1,98 +0,0 @@
package mtggoldfish
import (
"fmt"
"github.com/PuerkitoBio/goquery"
"github.com/geziyor/geziyor"
"github.com/geziyor/geziyor/client"
"github.com/geziyor/geziyor/export"
)
var DECK_SEARCH_URL string = "https://www.mtggoldfish.com/deck_searches/create"
const (
TypeMaindeck = "maindeck"
TypeSideboard = "sideboard"
TypeCommander = "commander"
)
type DeckSearchIncludedCard struct {
Name string
Quantity int
Type string
}
type DeckSearchParameters struct {
Name string
Format string
IncludeTournamentDecks bool
IncludeUserDecks bool
Player string
StartDate string // TODO: Date type?
EndDate string // TODO: Date type?
IncludedCards []DeckSearchCard
}
func (p *DeckSearchParameters) String() string {
searchUrl := DECK_SEARCH_URL + "?utf8=✓" +
"&deck_search[name]=" + p.Name +
"&deck_search[format]=" + p.Format +
"&deck_search[types][]="
if p.IncludeTournamentDecks {
searchUrl = searchUrl + "&deck_search[types][]=tournament"
}
if p.IncludeUserDecks {
searchUrl = searchUrl + "&deck_search[types][]=user"
}
searchUrl = searchUrl + "&deck_search[player]=" + p.Player
//"&deck_search[date_range]=" + p.StartDate + " - " + p.EndDate
return searchUrl
}
type DeckSearchCard struct {
Name string
Quantity int
Type string
}
type DeckSearchDecklist struct {
MtgGoldfishId int
Date string // TODO: Date type?
Name string
Source string
Format string
Author string
Cards []DeckSearchCard
}
type DeckSearchResults struct {
Decklists []DeckSearchDecklist
DeckCount int
PageCount int
}
func parseSearchResults(g *geziyor.Geziyor, r *client.Response) {
r.HTMLDoc.Find("div.table-responsive").Each(func(i int, s *goquery.Selection) {
g.Exports <- map[string]interface{}{
"test": s.Find("table>thead>tr>th").Text(),
}
})
}
func DeckSearch(params DeckSearchParameters) (DeckSearchResults, error) {
searchUrl := `https://www.mtggoldfish.com/deck_searches/create?utf8=✓&deck_search[name]=Burn&deck_search[format]=pauper&deck_search[types][]=&deck_search[types][]=tournament&deck_search[types][]=user&deck_search[player]=Jirach1&deck_search[date_range]=04%2F14%2F2024+-+04%2F28%2F2024&deck_search[deck_search_card_filters_attributes][0][card]=Kuldotha+Rebirth&deck_search[deck_search_card_filters_attributes][0][quantity]=4&deck_search[deck_search_card_filters_attributes][0][type]=maindeck&deck_search[deck_search_card_filters_attributes][1][card]=&deck_search[deck_search_card_filters_attributes][1][quantity]=1&deck_search[deck_search_card_filters_attributes][1][type]=maindeck&deck_search[deck_search_card_filters_attributes][2][card]=&deck_search[deck_search_card_filters_attributes][2][quantity]=1&deck_search[deck_search_card_filters_attributes][2][type]=maindeck&counter=3&commit=Search`
fmt.Println(searchUrl)
geziyor.NewGeziyor(&geziyor.Options{
StartURLs: []string{searchUrl},
ParseFunc: parseSearchResults,
Exporters: []export.Exporter{&export.JSON{}},
}).Start()
return DeckSearchResults{}, nil
}

View File

@ -4,76 +4,33 @@ import (
"encoding/json"
"errors"
"io"
"log"
"net/http"
"sevenkeys/scryfall/types"
)
var BULK_DATA_URI = "https://api.scryfall.com/bulk-data"
func GetBulkData() types.BulkDataList {
response, err := http.Get(BULK_DATA_URI)
func GetBulkDataByType(bulkDataType string) (types.BulkData, error) {
response, err := http.Get(BULK_DATA_URI + "/" + bulkDataType)
if err != nil {
log.Fatal(err)
return types.BulkData{}, err
}
if response.StatusCode != http.StatusOK {
log.Fatal(response.StatusCode)
return types.BulkData{}, errors.New("HTTP request failed with code: " + string(response.StatusCode))
}
defer response.Body.Close()
bulkDataBytes, err := io.ReadAll(response.Body)
if err != nil {
log.Fatal(err)
return types.BulkData{}, err
}
var bulkDataList types.BulkDataList
err = json.Unmarshal(bulkDataBytes, &bulkDataList)
var bulkData types.BulkData
err = json.Unmarshal(bulkDataBytes, &bulkData)
if err != nil {
log.Fatal(err)
return types.BulkData{}, err
}
return bulkDataList
}
func getBulkDownloadUri(bulkType string) (string, error) {
bulkDataList := GetBulkData()
for index := range bulkDataList.Data {
bulkData := bulkDataList.Data[index]
if bulkData.Type == bulkType {
return bulkData.DownloadUri, nil
}
}
return "", errors.New("No bulk data of type " + bulkType + " found.")
}
func GetAllCards() []types.Card {
allCardsUri, err := getBulkDownloadUri("all_cards")
if err != nil {
log.Fatal(err)
}
response, err := http.Get(allCardsUri)
if err != nil {
log.Fatal(err)
}
if response.StatusCode != http.StatusOK {
log.Fatal(response.StatusCode)
}
defer response.Body.Close()
allCardsBytes, err := io.ReadAll(response.Body)
if err != nil {
log.Fatal(err)
}
var allCards []types.Card
err = json.Unmarshal(allCardsBytes, &allCards)
if err != nil {
log.Fatal(err)
}
return allCards
return bulkData, nil
}

View File

@ -1,3 +1,36 @@
package methods
func GetSets() {}
import (
"encoding/json"
"errors"
"io"
"net/http"
"sevenkeys/scryfall/types"
)
const SETS_API_URL string = "https://api.scryfall.com/sets"
func GetSets() ([]types.Set, error) {
response, err := http.Get(SETS_API_URL)
if err != nil {
return []types.Set{}, nil
}
if response.StatusCode != http.StatusOK {
return []types.Set{}, errors.New("HTTP request failed with code: " + string(response.StatusCode))
}
defer response.Body.Close()
setsBytes, err := io.ReadAll(response.Body)
if err != nil {
return []types.Set{}, err
}
var setList types.SetList
err = json.Unmarshal(setsBytes, &setList)
if err != nil {
return []types.Set{}, err
}
return setList.Data, nil
}

View File

@ -1,5 +1,11 @@
package types
const BulkDataTypeOracleCards string = "oracle_cards"
const BulkDataTypeUniqueArtwork string = "unique_artwork"
const BulkDataTypeDefaultCards string = "default_cards"
const BulkDataTypeAllCards string = "all_cards"
const BulkDataTypeRulings string = "rulings"
type BulkData struct {
Id string `json:"id"`
Uri string `json:"uri"`
@ -12,9 +18,3 @@ type BulkData struct {
ContentType string `json:"content_type"`
ContentEncoding string `json:"content_encoding"`
}
type BulkDataList struct {
Object string `json:"object"`
HasMore bool `json:"has_more"`
Data []BulkData `json:"data"`
}

View File

@ -0,0 +1,15 @@
package types
type Card struct {
Id string `json:"id"` // GUID
Name string `json:"name"`
Set string `json:"set"`
Games []string `json:"games"`
Foil bool `json:"foil"`
NonFoil bool `json:"nonfoil"`
Reserved bool `json:"reserved"`
ContentWarning bool `json:"content_warning,omitempty"`
Promo bool `json:"promo"`
CollectorNumber string `json:"collector_number"`
Language string `json:"lang"`
}

View File

@ -1,130 +0,0 @@
package types
type Colors []string
type RelatedCard struct {
Id string `json:"id"`
Object string `json:"object"`
Component string `json:"component"`
Name string `json:"name"`
TypeLine string `json:"type_line"`
ApiUri string `json:"uri"`
}
type CardFace struct {
Artist string `json:"artist"`
ArtistId string `json:"artist_id"`
ManaValue float32 `json:"cmc"`
ColorIndicator Colors `json:"color_indicator"`
Colors Colors `json:"colors"`
Defense string `json:"defense"`
FlavorText string `json:"flavor_text"`
IllustrationId string `json:"illustration_id"`
ImageUris map[string]string `json:"image_uris"` // TODO: Find out the structure of this object
Layout string `json:"layout"`
Loyalty string `json:"loyalty"`
ManaCost string `json:"mana_cost"`
Name string `json:"name"`
Object string `json:"object"`
OracleId string `json:"oracle_id"`
Power string `json:"power"`
PrintedName string `json:"printed_name"`
PrintedText string `json:"printed_text"`
PrintedTypeLine string `json:"printed_type_line"`
Toughness string `json:"toughness"`
TypeLine string `json:"type_line"`
Watermark string `json:"watermark"`
}
type Card struct {
// Core fields
ArenaId int `json:"arena_id"`
Id string `json:"id"`
Language string `json:"lang"`
MtgoId int `json:"mtgo_id"`
MtgoFoilid int `json:"mtgo_foil_id"`
MultiverseIds []int `json:"multiverse_ids"`
TcgplayerId int `json:"tcgplayer_id"`
TcgplayerEtchedId int `json:"tcgplayer_etched_id"`
Object string `json:"object"`
Layout string `json:"layout"`
OracleId string `json:"oracle_id"`
PrintsSearchUri string `json:"prints_search_uri"`
RulingsUri string `json:"rulings_uri"`
ScryfallUri string `json:"scryfall_uri"`
ApiUri string `json:"uri"`
// Gameplay fields
AllParts []RelatedCard `json:"all_parts"`
CardFaces []CardFace `json:"card_faces"`
ManaValue float32 `json:"cmc"`
ColorIdentity Colors `json:"color_identity"`
ColorIndicator Colors `json:"color_indicator"`
Colors Colors `json:"colors"`
Defense string `json:"defense"`
EdhrecRank int `json:"edhrec_rank"`
HandModifier string `json:"hand_modifier"`
Keywords []string `json:"keywords"`
Legalities map[string]string `json:"legalities"`
LifeModifier string `json:"life_modifier"`
Loyalty string `json:"loyalty"`
ManaCost string `json:"mana_cost"`
Name string `json:"name"`
OracleText string `json:"oracle_text"`
PennyDreadfulRank int `json:"penny_rank"`
Power string `json:"power"`
ProducedMana Colors `json:"produced_mana"`
ReserveList bool `json:"reserved"`
Toughness string `json:"toughness"`
TypeLine string `json:"type_line"`
// Printing fields
Artist string `json:"artist"`
ArtistIds []string `json:"artist_ids"`
AttractionLights []interface{} `json:"attraction_lights"` // TODO: Figure out schema
Booster bool `json:"booster"`
BorderColor string `json:"border_color"`
CardBackId string `json:"card_back_id"`
CollectorNumber string `json:"collector_number"`
ContentWarning bool `json:"content_warning"`
Digital bool `json:"digital"`
Finishes []interface{} `json:"finishes"` // TODO: Find out how flags are formatted
FlavorName string `json:"flavor_name"`
FlavorText string `json:"flavor_text"`
FrameEffects []string `json:"frame_effects"`
Frame string `json:"frame"`
FullArt bool `json:"full_art"`
Games []string `json:"games"`
HighresImage bool `json:"highres_image"`
IllustrationId string `json:"illustration_id"`
ImageStatus string `json:"image_status"`
ImageUris map[string]string `json:"image_uris"` // TODO: Find out shape of object
Oversized bool `json:"oversized"`
Prices map[string]string `json:"prices"`
PrintedName string `json:"printed_name"`
PrintedText string `json:"printed_text"`
PrintedTypeLine string `json:"printed_type_line"`
Promo bool `json:"promo"`
PromoTypes []string `json:"promo_types"`
PurchaseUris interface{} `json:"purchase_uris"` // TODO: Find out shape of object
Rarity string `json:"rarity"`
RelatedUris interface{} `json:"related_uris"` // TODO: Find out shape of object
ReleasedAt string `json:"released_at"` // TODO: Datetime type?
Reprint bool `json:"reprint"`
ScryfallSetUri string `json:"scryfall_set_uri"`
SetName string `json:"set_name"`
SetSearchUri string `json:"set_search_uri"`
SetType string `json:"set_type"`
SetUri string `json:"set_uri"`
Set string `json:"set"`
SetId string `json:"set_id"`
StorySpotlight bool `json:"story_spotlight"`
Textless bool `json:"textless"`
Variation bool `json:"variation"`
VariationOf string `json:"variation_of"`
SecurityStamp string `json:"security_stamp"`
Watermark string `json:"watermark"`
PreviewedAt string `json:"preview.previewed_at"`
PreviewSourceUri string `json:"preview.source_uri"`
PreviewSource string `json:"preview.source"`
}

View File

@ -0,0 +1,15 @@
package types
type SetList struct {
Object string `json:"object"`
HasMore bool `json:"has_more"`
Data []Set `json:"data"`
}
type Set struct {
Code string `json:"code"`
Name string `json:"name"`
CardCount int `json:"card_count"`
IconSvgUri string `json:"icon_svg_uri"`
Digital bool `json:"digital"`
}

View File

@ -0,0 +1,3 @@
package types
const ScryfallTimestampFormat = "2006-01-02T15:04:05.999-07:00"

View File

@ -2,38 +2,28 @@ CREATE DATABASE IF NOT EXISTS sevenkeys;
USE sevenkeys;
CREATE TABLE IF NOT EXISTS Gamepiece (
Id INT AUTO_INCREMENT PRIMARY KEY,
Name VARCHAR(150) NOT NULL
CREATE TABLE IF NOT EXISTS CacheTimestamp (
CacheType ENUM('AllCardsBulkData') PRIMARY KEY,
Stamp DATETIME NOT NULL
);
CREATE TABLE IF NOT EXISTS ExpansionSet (
SetCode VARCHAR(6) PRIMARY KEY,
Name VARCHAR(60) NOT NULL,
CardCount INT NOT NULL,
IconSvgUri VARCHAR(60) NOT NULL
);
CREATE TABLE IF NOT EXISTS CardPrinting (
Id INT AUTO_INCREMENT PRIMARY KEY,
GamepieceId INT NOT NULL,
FOREIGN KEY (GamepieceId) REFERENCES Gamepiece(Id),
Id VARCHAR(36) PRIMARY KEY, -- GUID
Name VARCHAR(150) NOT NULL,
SetCode VARCHAR(6) NOT NULL,
ImageUrl VARCHAR(2048) NOT NULL
);
CREATE TABLE IF NOT EXISTS TournamentDecklist (
Id INT AUTO_INCREMENT PRIMARY KEY,
DeckName VARCHAR(100) NOT NULL,
DatePublished DATE NOT NULL,
Source VARCHAR(200) NOT NULL,
Format VARCHAR(25) NOT NULL,
AuthorName VARCHAR(100) NOT NULL
);
CREATE TABLE IF NOT EXISTS TournamentDecklistCard (
TournamentDecklistId INT NOT NULL,
GamepieceId INT NOT NULL,
PRIMARY KEY (TournamentDecklistId, GamepieceId),
FOREIGN KEY TournamentDecklistId REFERENCES TournamentDecklist(Id),
FOREIGN KEY GamepieceId REFERENCES Gamepiece(Id),
CardPrintingId INT NULL,
FOREIGN KEY CardPrintingId REFERENCES CardPrinting(Id),
Quantity INT NOT NULL
FOREIGN KEY (SetCode) REFERENCES ExpansionSet(SetCode),
HasFoil BOOLEAN NOT NULL,
HasNonFoil BOOLEAN NOT NULL,
IsReserved BOOLEAN NOT NULL,
IsRacist BOOLEAN NOT NULL,
IsPromo BOOLEAN NOT NULL,
CollectorNumber VARCHAR(10) NOT NULL,
Language VARCHAR(3) NOT NULL
);

1
sevenkeys/test.json Normal file
View File

@ -0,0 +1 @@
{"object":"card","id":"0000579f-7b35-4ed3-b44c-db2a538066fe","oracle_id":"44623693-51d6-49ad-8cd7-140505caf02f","multiverse_ids":[109722],"mtgo_id":25527,"mtgo_foil_id":25528,"tcgplayer_id":14240,"cardmarket_id":13850,"name":"Fury Sliver","lang":"en","released_at":"2006-10-06","uri":"https://api.scryfall.com/cards/0000579f-7b35-4ed3-b44c-db2a538066fe","scryfall_uri":"https://scryfall.com/card/tsp/157/fury-sliver?utm_source=api","layout":"normal","highres_image":true,"image_status":"highres_scan","image_uris":{"small":"https://cards.scryfall.io/small/front/0/0/0000579f-7b35-4ed3-b44c-db2a538066fe.jpg?1562894979","normal":"https://cards.scryfall.io/normal/front/0/0/0000579f-7b35-4ed3-b44c-db2a538066fe.jpg?1562894979","large":"https://cards.scryfall.io/large/front/0/0/0000579f-7b35-4ed3-b44c-db2a538066fe.jpg?1562894979","png":"https://cards.scryfall.io/png/front/0/0/0000579f-7b35-4ed3-b44c-db2a538066fe.png?1562894979","art_crop":"https://cards.scryfall.io/art_crop/front/0/0/0000579f-7b35-4ed3-b44c-db2a538066fe.jpg?1562894979","border_crop":"https://cards.scryfall.io/border_crop/front/0/0/0000579f-7b35-4ed3-b44c-db2a538066fe.jpg?1562894979"},"mana_cost":"{5}{R}","cmc":6.0,"type_line":"Creature — Sliver","oracle_text":"All Sliver creatures have double strike.","power":"3","toughness":"3","colors":["R"],"color_identity":["R"],"keywords":[],"legalities":{"standard":"not_legal","future":"not_legal","historic":"not_legal","timeless":"not_legal","gladiator":"not_legal","pioneer":"not_legal","explorer":"not_legal","modern":"legal","legacy":"legal","pauper":"not_legal","vintage":"legal","penny":"not_legal","commander":"legal","oathbreaker":"legal","standardbrawl":"not_legal","brawl":"not_legal","alchemy":"not_legal","paupercommander":"restricted","duel":"legal","oldschool":"not_legal","premodern":"not_legal","predh":"legal"},"games":["paper","mtgo"],"reserved":false,"foil":true,"nonfoil":true,"finishes":["nonfoil","foil"],"oversized":false,"promo":false,"reprint":false,"variation":false,"set_id":"c1d109bc-ffd8-428f-8d7d-3f8d7e648046","set":"tsp","set_name":"Time Spiral","set_type":"expansion","set_uri":"https://api.scryfall.com/sets/c1d109bc-ffd8-428f-8d7d-3f8d7e648046","set_search_uri":"https://api.scryfall.com/cards/search?order=set&q=e%3Atsp&unique=prints","scryfall_set_uri":"https://scryfall.com/sets/tsp?utm_source=api","rulings_uri":"https://api.scryfall.com/cards/0000579f-7b35-4ed3-b44c-db2a538066fe/rulings","prints_search_uri":"https://api.scryfall.com/cards/search?order=released&q=oracleid%3A44623693-51d6-49ad-8cd7-140505caf02f&unique=prints","collector_number":"157","digital":false,"rarity":"uncommon","flavor_text":"\"A rift opened, and our arrows were abruptly stilled. To move was to push the world. But the sliver's claw still twitched, red wounds appeared in Thed's chest, and ribbons of blood hung in the air.\"\n—Adom Capashen, Benalish hero","card_back_id":"0aeebaf5-8c7d-4636-9e82-8c27447861f7","artist":"Paolo Parente","artist_ids":["d48dd097-720d-476a-8722-6a02854ae28b"],"illustration_id":"2fcca987-364c-4738-a75b-099d8a26d614","border_color":"black","frame":"2003","full_art":false,"textless":false,"booster":true,"story_spotlight":false,"edhrec_rank":7361,"penny_rank":10143,"prices":{"usd":"0.40","usd_foil":"1.80","usd_etched":null,"eur":"0.19","eur_foil":"1.02","tix":"0.03"},"related_uris":{"gatherer":"https://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid=109722&printed=false","tcgplayer_infinite_articles":"https://tcgplayer.pxf.io/c/4931599/1830156/21018?subId1=api&trafcat=infinite&u=https%3A%2F%2Finfinite.tcgplayer.com%2Fsearch%3FcontentMode%3Darticle%26game%3Dmagic%26partner%3Dscryfall%26q%3DFury%2BSliver","tcgplayer_infinite_decks":"https://tcgplayer.pxf.io/c/4931599/1830156/21018?subId1=api&trafcat=infinite&u=https%3A%2F%2Finfinite.tcgplayer.com%2Fsearch%3FcontentMode%3Ddeck%26game%3Dmagic%26partner%3Dscryfall%26q%3DFury%2BSliver","edhrec":"https://edhrec.com/route/?cc=Fury+Sliver"},"purchase_uris":{"tcgplayer":"https://tcgplayer.pxf.io/c/4931599/1830156/21018?subId1=api&u=https%3A%2F%2Fwww.tcgplayer.com%2Fproduct%2F14240%3Fpage%3D1","cardmarket":"https://www.cardmarket.com/en/Magic/Products/Singles/Time-Spiral/Fury-Sliver?referrer=scryfall&utm_campaign=card_prices&utm_medium=text&utm_source=scryfall","cardhoarder":"https://www.cardhoarder.com/cards/25527?affiliate_id=scryfall&ref=card-profile&utm_campaign=affiliate&utm_medium=card&utm_source=scryfall"}},