Cache bulk data file
This commit is contained in:
parent
6def107873
commit
c8133803f7
|
@ -1 +1,2 @@
|
||||||
|
cache/
|
||||||
cmd/database/config.json
|
cmd/database/config.json
|
||||||
|
|
|
@ -1,13 +1,82 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
"sevenkeys/database"
|
"sevenkeys/database"
|
||||||
|
"sevenkeys/database/entities"
|
||||||
"sevenkeys/database/operations"
|
"sevenkeys/database/operations"
|
||||||
"sevenkeys/scryfall/methods"
|
"sevenkeys/scryfall/methods"
|
||||||
|
"sevenkeys/scryfall/types"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func check(err error) {
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
log.Printf("Getting bulk data listing...")
|
||||||
|
allCardsBulkData, err := methods.GetBulkDataByType(types.BulkDataTypeAllCards)
|
||||||
|
check(err)
|
||||||
|
log.Printf("Got bulk data listing.")
|
||||||
|
|
||||||
|
log.Printf("Checking if we need to download bulk data...")
|
||||||
|
updatedAtTimestamp, err := time.Parse(types.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt)
|
||||||
|
check(err)
|
||||||
|
// Remove the fractional seconds component from the downloaded timestamp because it isn't saved in the cache table
|
||||||
|
updatedAtTimestamp = updatedAtTimestamp.Truncate(time.Second)
|
||||||
|
|
||||||
|
db := database.GetDatabaseFromConfig("config.json")
|
||||||
|
|
||||||
|
cachedFileTimestamp, err := operations.GetCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData)
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
if updatedAtTimestamp.After(cachedFileTimestamp) {
|
||||||
|
log.Printf("Cached time: ", cachedFileTimestamp)
|
||||||
|
log.Printf("Update time: ", updatedAtTimestamp)
|
||||||
|
log.Printf("Bulk data has been updated since last cache, redownloading.")
|
||||||
|
cacheDir := "cache"
|
||||||
|
|
||||||
|
err = os.RemoveAll(cacheDir)
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
err := os.Mkdir(cacheDir, os.ModePerm)
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
log.Printf("Downloading bulk card data...")
|
||||||
|
bulkCardsResponse, err := http.Get(allCardsBulkData.DownloadUri)
|
||||||
|
check(err)
|
||||||
|
log.Printf("Downloaded bulk card data.")
|
||||||
|
|
||||||
|
log.Printf("Writing card data to cache file...")
|
||||||
|
cacheFilename := cacheDir + "/all-cards.json"
|
||||||
|
cacheFile, err := os.Create(cacheFilename)
|
||||||
|
check(err)
|
||||||
|
|
||||||
|
defer bulkCardsResponse.Body.Close()
|
||||||
|
defer cacheFile.Close()
|
||||||
|
io.Copy(cacheFile, bulkCardsResponse.Body)
|
||||||
|
log.Printf("Cache file written.")
|
||||||
|
|
||||||
|
log.Printf("Recording timestamp...")
|
||||||
|
err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp)
|
||||||
|
check(err)
|
||||||
|
log.Printf("Timestamp recorded.")
|
||||||
|
} else {
|
||||||
|
log.Printf("Bulk data has not been updated. Skipping download.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Marshal the bulk cards json from the cache file into a struct
|
||||||
|
// Import the json into the database
|
||||||
|
|
||||||
|
/* OLD CODE
|
||||||
db := database.GetDatabaseFromConfig("config.json")
|
db := database.GetDatabaseFromConfig("config.json")
|
||||||
allCards := methods.GetAllCards()
|
allCards := methods.GetAllCards()
|
||||||
operations.InsertCards(db, allCards)
|
operations.InsertCards(db, allCards)
|
||||||
|
*/
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,3 @@
|
||||||
|
package entities
|
||||||
|
|
||||||
|
const CacheTypeAllCardsBulkData = "AllCardsBulkData"
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"log"
|
"log"
|
||||||
"sevenkeys/scryfall/types"
|
"sevenkeys/scryfall/types"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
var ErrGamepieceExists error = errors.New("Gamepiece already exists in database")
|
var ErrGamepieceExists error = errors.New("Gamepiece already exists in database")
|
||||||
|
@ -56,3 +57,20 @@ func InsertCards(db *sql.DB, cards []types.Card) {
|
||||||
insertCardPrinting(db, gamepieceId, card.Set, card.ImageUris["png"])
|
insertCardPrinting(db, gamepieceId, card.Set, card.ImageUris["png"])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func InsertOrUpdateCacheTimestampByType(db *sql.DB, cacheType string, stamp time.Time) error {
|
||||||
|
query := `INSERT INTO CacheTimestamps (CacheType, Stamp) VALUES (?, ?) ON DUPLICATE KEY UPDATE Stamp = ?;`
|
||||||
|
|
||||||
|
insertOrUpdate, err := db.Prepare(query)
|
||||||
|
defer insertOrUpdate.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = insertOrUpdate.Exec(cacheType, stamp, stamp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@ package operations
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"sevenkeys/database/entities"
|
"sevenkeys/database/entities"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) {
|
func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) {
|
||||||
|
@ -13,3 +14,21 @@ func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) {
|
||||||
|
|
||||||
return gamepiece, err
|
return gamepiece, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func GetCacheTimestampByType(db *sql.DB, cacheType string) (time.Time, error) {
|
||||||
|
var timestamp string
|
||||||
|
|
||||||
|
query := "SELECT Stamp FROM CacheTimestamps WHERE CacheType = ?;"
|
||||||
|
err := db.QueryRow(query, cacheType).Scan(×tamp)
|
||||||
|
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
return time.Unix(0, 0), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
stamp, err := time.Parse("2006-01-02 15:04:05", timestamp)
|
||||||
|
if err != nil {
|
||||||
|
return time.Unix(0, 0), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return stamp, err
|
||||||
|
}
|
||||||
|
|
|
@ -4,76 +4,58 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"sevenkeys/scryfall/types"
|
"sevenkeys/scryfall/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
var BULK_DATA_URI = "https://api.scryfall.com/bulk-data"
|
var BULK_DATA_URI = "https://api.scryfall.com/bulk-data"
|
||||||
|
|
||||||
func GetBulkData() types.BulkDataList {
|
func GetBulkData() (types.BulkDataList, error) {
|
||||||
response, err := http.Get(BULK_DATA_URI)
|
response, err := http.Get(BULK_DATA_URI)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
return types.BulkDataList{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if response.StatusCode != http.StatusOK {
|
if response.StatusCode != http.StatusOK {
|
||||||
log.Fatal(response.StatusCode)
|
return types.BulkDataList{}, errors.New("HTTP request failed with code: " + string(response.StatusCode))
|
||||||
}
|
}
|
||||||
|
|
||||||
defer response.Body.Close()
|
defer response.Body.Close()
|
||||||
bulkDataBytes, err := io.ReadAll(response.Body)
|
bulkDataBytes, err := io.ReadAll(response.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
return types.BulkDataList{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var bulkDataList types.BulkDataList
|
var bulkDataList types.BulkDataList
|
||||||
err = json.Unmarshal(bulkDataBytes, &bulkDataList)
|
err = json.Unmarshal(bulkDataBytes, &bulkDataList)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
return types.BulkDataList{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return bulkDataList
|
return bulkDataList, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getBulkDownloadUri(bulkType string) (string, error) {
|
func GetBulkDataByType(bulkDataType string) (types.BulkData, error) {
|
||||||
bulkDataList := GetBulkData()
|
response, err := http.Get(BULK_DATA_URI + "/" + bulkDataType)
|
||||||
for index := range bulkDataList.Data {
|
|
||||||
bulkData := bulkDataList.Data[index]
|
|
||||||
if bulkData.Type == bulkType {
|
|
||||||
return bulkData.DownloadUri, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", errors.New("No bulk data of type " + bulkType + " found.")
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetAllCards() []types.Card {
|
|
||||||
allCardsUri, err := getBulkDownloadUri("all_cards")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
return types.BulkData{}, err
|
||||||
}
|
|
||||||
|
|
||||||
response, err := http.Get(allCardsUri)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if response.StatusCode != http.StatusOK {
|
if response.StatusCode != http.StatusOK {
|
||||||
log.Fatal(response.StatusCode)
|
return types.BulkData{}, errors.New("HTTP request failed with code: " + string(response.StatusCode))
|
||||||
}
|
}
|
||||||
|
|
||||||
defer response.Body.Close()
|
defer response.Body.Close()
|
||||||
allCardsBytes, err := io.ReadAll(response.Body)
|
bulkDataBytes, err := io.ReadAll(response.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
return types.BulkData{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var allCards []types.Card
|
var bulkData types.BulkData
|
||||||
err = json.Unmarshal(allCardsBytes, &allCards)
|
err = json.Unmarshal(bulkDataBytes, &bulkData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
return types.BulkData{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return allCards
|
return bulkData, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,11 @@
|
||||||
package types
|
package types
|
||||||
|
|
||||||
|
const BulkDataTypeOracleCards string = "oracle_cards"
|
||||||
|
const BulkDataTypeUniqueArtwork string = "unique_artwork"
|
||||||
|
const BulkDataTypeDefaultCards string = "default_cards"
|
||||||
|
const BulkDataTypeAllCards string = "all_cards"
|
||||||
|
const BulkDataTypeRulings string = "rulings"
|
||||||
|
|
||||||
type BulkData struct {
|
type BulkData struct {
|
||||||
Id string `json:"id"`
|
Id string `json:"id"`
|
||||||
Uri string `json:"uri"`
|
Uri string `json:"uri"`
|
||||||
|
|
|
@ -0,0 +1,3 @@
|
||||||
|
package types
|
||||||
|
|
||||||
|
const ScryfallTimestampFormat = "2006-01-02T15:04:05.999-07:00"
|
|
@ -3,7 +3,7 @@ CREATE DATABASE IF NOT EXISTS sevenkeys;
|
||||||
USE sevenkeys;
|
USE sevenkeys;
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS CacheTimestamps (
|
CREATE TABLE IF NOT EXISTS CacheTimestamps (
|
||||||
CacheType ENUM('BulkCardPrintings') PRIMARY KEY,
|
CacheType ENUM('AllCardsBulkData') PRIMARY KEY,
|
||||||
Stamp DATETIME NOT NULL
|
Stamp DATETIME NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue