From 8b7ec980f49f90b86cd88f58bc8a684660ad360a Mon Sep 17 00:00:00 2001 From: The Magician Date: Tue, 28 May 2024 15:29:13 +0100 Subject: [PATCH] Check if update is required --- sevenkeys/database/cachetimestamp.go | 14 ++ sevenkeys/database/entities/cachetimestamp.go | 3 - sevenkeys/database/operations/inserts.go | 2 +- sevenkeys/database/operations/selects.go | 28 --- sevenkeys/importer.go | 175 ++++++++++++++++++ sevenkeys/logic/cache.go | 22 +++ sevenkeys/logic/errcheck.go | 9 + sevenkeys/logic/updatecheck.go | 32 ++++ sevenkeys/main.go | 24 +++ sevenkeys/printinglist.go | 27 +++ 10 files changed, 304 insertions(+), 32 deletions(-) create mode 100644 sevenkeys/database/cachetimestamp.go delete mode 100644 sevenkeys/database/entities/cachetimestamp.go create mode 100644 sevenkeys/importer.go create mode 100644 sevenkeys/logic/cache.go create mode 100644 sevenkeys/logic/errcheck.go create mode 100644 sevenkeys/logic/updatecheck.go create mode 100644 sevenkeys/main.go create mode 100644 sevenkeys/printinglist.go diff --git a/sevenkeys/database/cachetimestamp.go b/sevenkeys/database/cachetimestamp.go new file mode 100644 index 0000000..7910059 --- /dev/null +++ b/sevenkeys/database/cachetimestamp.go @@ -0,0 +1,14 @@ +package database + +import "database/sql" + +const CacheTypeAllCardsBulkData = "AllCardsBulkData" + +func GetCacheTimestampByType(db *sql.DB, cacheType string) (string, error) { + var timestamp string + + query := "SELECT Stamp FROM CacheTimestamp WHERE CacheType = ?;" + err := db.QueryRow(query, cacheType).Scan(×tamp) + + return timestamp, err +} diff --git a/sevenkeys/database/entities/cachetimestamp.go b/sevenkeys/database/entities/cachetimestamp.go deleted file mode 100644 index 7871b00..0000000 --- a/sevenkeys/database/entities/cachetimestamp.go +++ /dev/null @@ -1,3 +0,0 @@ -package entities - -const CacheTypeAllCardsBulkData = "AllCardsBulkData" diff --git a/sevenkeys/database/operations/inserts.go b/sevenkeys/database/operations/inserts.go index 0b67742..badb025 100644 --- a/sevenkeys/database/operations/inserts.go +++ b/sevenkeys/database/operations/inserts.go @@ -2,7 +2,7 @@ package operations import ( "database/sql" - "sevenkeys/scryfall/types" + "sevenkeys/logic/scryfall/types" "time" ) diff --git a/sevenkeys/database/operations/selects.go b/sevenkeys/database/operations/selects.go index 0758d47..22191a2 100644 --- a/sevenkeys/database/operations/selects.go +++ b/sevenkeys/database/operations/selects.go @@ -5,36 +5,8 @@ import ( "fmt" "sevenkeys/database/entities" "strings" - "time" ) -func GetGamepieceByName(db *sql.DB, name string) (entities.Gamepiece, error) { - var gamepiece entities.Gamepiece - - query := "SELECT Id, Name FROM Gamepiece WHERE Name = ?;" - err := db.QueryRow(query, name).Scan(&gamepiece.Id, &gamepiece.Name) - - return gamepiece, err -} - -func GetCacheTimestampByType(db *sql.DB, cacheType string) (time.Time, error) { - var timestamp string - - query := "SELECT Stamp FROM CacheTimestamp WHERE CacheType = ?;" - err := db.QueryRow(query, cacheType).Scan(×tamp) - - if err == sql.ErrNoRows { - return time.Unix(0, 0), nil - } - - stamp, err := time.Parse("2006-01-02 15:04:05", timestamp) - if err != nil { - return time.Unix(0, 0), err - } - - return stamp, err -} - func GetCardSearchOptions(db *sql.DB) ([]string, error) { var searchOptions []string diff --git a/sevenkeys/importer.go b/sevenkeys/importer.go new file mode 100644 index 0000000..a01f288 --- /dev/null +++ b/sevenkeys/importer.go @@ -0,0 +1,175 @@ +package main + +import ( + "database/sql" + "encoding/json" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "sevenkeys/database/entities" + "sevenkeys/database/operations" + "sevenkeys/logic/scryfall/types" + "time" +) + +const CACHE_DIR string = "cache" + +const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/" +const SET_ICON_FILE_EXTENSION string = ".svg" + +const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json" + +const GAME_PAPER = "paper" + +func check(err error) { + if err != nil { + log.Fatal(err) + } +} + +func createCacheDirectories() error { + err := os.Mkdir(CACHE_DIR, os.ModePerm) + if err != nil && !os.IsExist(err) { + return err + } + + err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm) + if err != nil && !os.IsExist(err) { + return err + } + + return nil +} + +func importSets(db *sql.DB, sets []types.Set) error { + for _, set := range sets { + // We're only interested in paper cards, so skip importing + // any sets that were only released in a video game + if set.Digital { + continue + } + + log.Println("Importing " + set.Code) + err := operations.InsertOrUpdateSet(db, set) + if err != nil { + return err + } + + log.Println("Downloading logo for " + set.Code) + response, err := http.Get(set.IconSvgUri) + defer response.Body.Close() + if err != nil { + return err + } + + iconFilename := SET_ICON_CACHE_DIR + set.Code + SET_ICON_FILE_EXTENSION + iconFile, err := os.Create(iconFilename) + if err != nil { + return err + } + + io.Copy(iconFile, response.Body) + log.Println("Finished importing " + set.Code) + } + + return nil +} + +func cacheAllCardsFile(db *sql.DB, uri string, updatedAtTimestamp time.Time) error { + log.Printf("Downloading bulk card data...") + bulkCardsResponse, err := http.Get(uri) + if err != nil { + return err + } + log.Printf("Downloaded bulk card data.") + + log.Printf("Writing card data to cache file...") + cacheFile, err := os.Create(ALL_CARDS_CACHE_FILENAME) + if err != nil { + return err + } + + defer bulkCardsResponse.Body.Close() + defer cacheFile.Close() + io.Copy(cacheFile, bulkCardsResponse.Body) + log.Printf("Cache file written.") + + log.Printf("Recording timestamp...") + err = operations.InsertOrUpdateCacheTimestampByType(db, entities.CacheTypeAllCardsBulkData, updatedAtTimestamp) + if err != nil { + return err + } + log.Printf("Timestamp recorded.") + + return nil +} + +func importCards(db *sql.DB, cards []types.Card) error { + for _, card := range cards { + // We're only interested in paper cards, so skip cards or printings of a card which + // aren't available in paper + if !isPaper(card) { + continue + } + + err := operations.InsertCard(db, card) + if err != nil { + return err + } + } + + return nil +} + +func isPaper(card types.Card) bool { + var paper bool = false + + for _, game := range card.Games { + if game == GAME_PAPER { + paper = true + } + } + + return paper +} + +func main() { + log.Println("Creating cache directories...") + err := createCacheDirectories() + check(err) + log.Println("Created cache directories.") + + log.Println("Downloading set data from Scryfall...") + sets, err := methods.GetSets() + check(err) + log.Println("Downloaded set data.") + + log.Println("Importing set data...") + err = importSets(db, sets) + check(err) + log.Println("Imported sets.") + + if updated { + log.Printf("Bulk data has been updated since last cache, redownloading.") + err = cacheAllCardsFile(db, allCardsBulkData.DownloadUri, updatedAtTimestamp) + check(err) + } else { + log.Printf("Bulk data has not been updated. Skipping download.") + } + + log.Printf("Unmarsaling file into slice...") + allCardsBytes, err := ioutil.ReadFile(ALL_CARDS_CACHE_FILENAME) + check(err) + + var allCards []types.Card + err = json.Unmarshal(allCardsBytes, &allCards) + check(err) + log.Printf("Unmarshaled file.") + + log.Printf("Importing card data into database...") + err = importCards(db, allCards) + check(err) + log.Printf("Imported card data.") +} diff --git a/sevenkeys/logic/cache.go b/sevenkeys/logic/cache.go new file mode 100644 index 0000000..6b63655 --- /dev/null +++ b/sevenkeys/logic/cache.go @@ -0,0 +1,22 @@ +package logic + +import "os" + +const CACHE_DIR string = "cache" +const SET_ICON_CACHE_DIR string = CACHE_DIR + "/seticons/" +const SET_ICON_FILE_EXTENSION string = ".svg" +const ALL_CARDS_CACHE_FILENAME = CACHE_DIR + "/all-cards.json" + +func CreateCacheDirectories() error { + err := os.Mkdir(CACHE_DIR, os.ModePerm) + if err != nil && !os.IsExist(err) { + return err + } + + err = os.Mkdir(SET_ICON_CACHE_DIR, os.ModePerm) + if err != nil && !os.IsExist(err) { + return err + } + + return nil +} diff --git a/sevenkeys/logic/errcheck.go b/sevenkeys/logic/errcheck.go new file mode 100644 index 0000000..27edab6 --- /dev/null +++ b/sevenkeys/logic/errcheck.go @@ -0,0 +1,9 @@ +package logic + +import "log" + +func Check(err error) { + if err != nil { + log.Fatal(err) + } +} diff --git a/sevenkeys/logic/updatecheck.go b/sevenkeys/logic/updatecheck.go new file mode 100644 index 0000000..8aaf522 --- /dev/null +++ b/sevenkeys/logic/updatecheck.go @@ -0,0 +1,32 @@ +package logic + +import ( + "database/sql" + "sevenkeys/database" + "sevenkeys/logic/scryfall" + "time" +) + +func CheckForUpdates(db *sql.DB) (bool, error) { + cachedFileTimestampStr, err := database.GetCacheTimestampByType(db, database.CacheTypeAllCardsBulkData) + if err == sql.ErrNoRows { + return true, nil + } else if err != nil { + return false, err + } + + cachedFileTimestamp, err := time.Parse("2006-01-02 15:04:05", cachedFileTimestampStr) + + allCardsBulkData, err := scryfall.GetBulkDataByType(scryfall.BulkDataTypeAllCards) + if err != nil { + return false, err + } + + bulkCardsUpdatedTimestamp, err := time.Parse(scryfall.ScryfallTimestampFormat, allCardsBulkData.UpdatedAt) + if err != nil { + return false, err + } + bulkCardsUpdatedTimestamp = bulkCardsUpdatedTimestamp.Truncate(time.Second) + + return bulkCardsUpdatedTimestamp.After(cachedFileTimestamp), nil +} diff --git a/sevenkeys/main.go b/sevenkeys/main.go new file mode 100644 index 0000000..0d86d3f --- /dev/null +++ b/sevenkeys/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "fmt" + "sevenkeys/database" + "sevenkeys/logic" +) + +func main() { + db := database.GetDatabaseFromConfig("config.json") + + needsUpdate, err := logic.CheckForUpdates(db) + logic.Check(err) + + if needsUpdate { + fmt.Println("Needs update") + /* + if logic.AskForScryfallUpdate() { + err = logic.RunScryfallUpdate() + logic.Check(err) + } + */ + } +} diff --git a/sevenkeys/printinglist.go b/sevenkeys/printinglist.go new file mode 100644 index 0000000..2af5605 --- /dev/null +++ b/sevenkeys/printinglist.go @@ -0,0 +1,27 @@ +package main + +/* +func main() { + db := database.GetDatabaseFromConfig("config.json") + cardSearchOptions, err := operations.GetCardSearchOptions(db) + check(err) + + cmd := exec.Command("fzf") + cmd.Stderr = os.Stderr + + fzfStdin, err := cmd.StdinPipe() + check(err) + + go func() { + defer fzfStdin.Close() + for _, option := range cardSearchOptions { + io.WriteString(fzfStdin, option+"\n") + } + }() + + fzfOutput, err := cmd.Output() + check(err) + + fmt.Println("Output:", string(fzfOutput)) +} +*/