From 63e4cef7d709a5af9bbbeb80fe77a93db7d80ace Mon Sep 17 00:00:00 2001
From: The Magician <themagician@engineer.com>
Date: Mon, 11 Dec 2023 18:30:58 +0000
Subject: [PATCH] Initial commit

---
 .../__pycache__/downloader.cpython-311.pyc    | Bin 0 -> 3717 bytes
 .../downloader/downloader.py                  |  61 ++++++
 penny_dreadful_downloader/downloader/runtest  |   2 +
 .../downloader/test_downloader.py             | 175 ++++++++++++++++++
 4 files changed, 238 insertions(+)
 create mode 100644 penny_dreadful_downloader/downloader/__pycache__/downloader.cpython-311.pyc
 create mode 100755 penny_dreadful_downloader/downloader/downloader.py
 create mode 100755 penny_dreadful_downloader/downloader/runtest
 create mode 100644 penny_dreadful_downloader/downloader/test_downloader.py

diff --git a/penny_dreadful_downloader/downloader/__pycache__/downloader.cpython-311.pyc b/penny_dreadful_downloader/downloader/__pycache__/downloader.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6c7da0e0262b60d3dd223abbd55343ce4dc394d2
GIT binary patch
literal 3717
zcmb_f-%lIK9iROLw%2y~;TTgOfg}`60wy%1xl5En0=diO!bKs~DfH-S@ovDmwav{e
zAy6Z^(mkLmx}sC0@X!bMbeH6mm#C`Rs87BBU>B_vYbB&eeY%&*Rr(O!Q@=B|*FQ)j
zozyY&*_qj|XU3oJ=lh-EaVQi(P`*6zFHLJj=%2JxEqtZ%Y7-g{k%lzRKnvw3x4=Q4
zH~6%$AaF<%7DP?_94$zi1lXdr0JdtafIiI!=-2#!0WAP1YcgO^4{5<q*%~;=BR%*C
zR(V=o8^N61Jgf20_jGhF&bgtBy1HhjZm${h*|pWpn~9ZdT2I_fB~#fol1$;Gp3t)Q
z6010SM^BncS~XYnw7LxM)eK3j>Y2=jqG4UtV1|UDFRMn%-0&KTrkZLZ>1_cTB2)qD
ztD681k;%cMQto(dje7*`r(S;xnNrmXp&YtHXJAF+AHffPS|t|Eqqs1)IkaM$tK{88
zd4tvcv3-vvv*`rURg#7MZ^|oX+8A3^G0|~cbbVO=Y)vO7afM~w#B>!`H1&HXFeR?4
z%Q~T3>i^+4{~wyP?0Uw?s+x|0<Z5$lb;CVW?JAX>Ds-Qi3d;|b90$mweYx}gc`G(;
z%NHH_Vt!^{j_pSN*j>10%M*?~VTlv3zz!un8lgjf5~JUM#zRUaJj+Gn%G$}sH6F&j
zZxuXptCsZHsI9vw8`gySZ)tqDXMUHuMVxb631}#l&A3w1PziA*VyY&I3$B<^)4D6E
zriqCweyVQ}AY3rj<uc!x{%F^qA{9Dt1o|PU<Q;%KD$3#eXLmOC<bF$DDt!D*_>2E<
z;XjA%@hN9~${L)4(SLkwU7vTZ-?A1|=lYU0yJU$=ScY#mu;YpI3wT4>DZuk!IUr9i
z$JO}S;_Jm)FV$R=SPOK$14R>>#4qH8GJpKa#Ae{v<D>5^{`S<#i7iYW6PH~7?Nmm)
zw30G3w}qHEm05OqtV3H@adO3#mNA5OT*RGV?zXHzu<F>A%TI->+YOavrexOAOFAZW
zTm4YVythLrREVz_;3Htr29^900Qe8J{bBL<i@T@oP`?xE&tEQf#2$?1ukJ^p53cMQ
zPp;aLvrgpfi^zq&$c5*h*^y}{GM&Fxlw+0{dy5x#VD=y9g?iO(@j1Tcl{=n_HKC@=
zE#S7`B@<ZAE1IZDjj4s&f*)GuT;521?*mI$T*3NnS7O-)I*@cAi3B}&U_Ik<DsHEy
ztyo_*)TCbZL%>K8Q%RY6nz(H&aVkj_YlIECZ54-<ht{ys<daC9PdpoZ5*CU=B_9HS
zTiPQp+6VU92MWjS_93TzDE~pRYoIXjV9OFC#n`YD8*yUe&!SH3EFI_?deN2G>q<Op
zeIB*Drkt)RIx6>AV$WNA^!7+GJ=P&(Kt`!c>0AndhqxKEIJhO$M@+6+|Dp-!Xtf-8
zE9ImdpA*l6!Tq1*_<8g#H%C)B^@5@r21^!9v5cR1iz^a{2Uo68OtXyFp-j7h+8cZr
zro|@!NDBZBifUNMH5U+k8eaBMt6u>C^BvKLqhF4G<F`9ToQ{$F)t4O+fapL83DOw`
z+kyR9{K<$Nn{;B6+oAoQ#Iu0ibI$2GXZd2q!O3lT=bG(1YWa@7l%tjyeTxh3!kYg(
zOd1n4(0%L^aLN`3X{t{6weWx>)#xaYeJzRQP>y3%AJ0Kj$78TGkS#GCV|qqo$`hsz
zgK<ds?4V^ddb|S3ZlJ7M8U#(!Za(`IW`p0QGeRZP0Ko9z$;XRdE&fUQS}7r3I>o^D
z4Qz*YHun$pJ@}+BVjmiH4vlUHiUTLUKC<1q6S94WE#Kjnaz}pdAe(f@>|k`D@gvy0
z;jvjKb&aDJyc|P~r|o8mLy`yk+P!5GZHYPY&8eu_BXE&ppT-$Dl#!32Q;3M`{5|0o
zTIV=)3od;ex<9Yg4q)mJWwTAGOF3gE90A-qWTrE)Q#U$4J3ar=oO1cfhqLp_%#|Cn
zmu`M^<CZIC;kxHa>lmC(PaUk2Rk%dyq#PvlD5g<o=<G1BIGFYc)ddZl%7N0srl=k%
zW%tg2$v9N<R{-kXvB%0+iamJ78D#Mir$H01I=D0b<ci%n`8;HI&RXv;*qsa8zWq@9
z&hYM~@)1-*oIhUppcst42=?v;dv_P?;II=Mwt~YY0ba594E{|m|3jfR_NLbB*RS6=
zT#{g4Y;%O>`M*b}d!_Gs#p!<kcgHxuBn*0m3XF*?2aU$dZ)2Fkg_qh@O71}JY{-|*
zmu0>oUp9p^6^|FV>_*9w7mF`Uy$}~Lbp-wefGaWY5V{x7ePUWoW$=42MCC%x0noc>
zzbzki<fCv64R`Mzx5G!A@R9s``@w<2kR2Rxf+P9aeKESTW{Z7}*k@J$SR}eW1^%^2
zWfUbI#`JV%sl*kQ$Z2*7#~0`rz5X_3^QOJCUD3&?1Qq35#AavLm>+!S(^+lJ&@W(m
zI?~%Mu>e3099L>VTp;hKpCbB6o_&g_&uaX8*`2jev?TJ}&mrAZ!N+k)4x~}T7Ln^N
zArFEQ*IPm!gacex33(8cxK8@E0%4H@HF*#exx=(ufglot+nakrw<UBJMPGjAcb7lE
QoM)esfW)7|F#E;-1yOxSi~s-t

literal 0
HcmV?d00001

diff --git a/penny_dreadful_downloader/downloader/downloader.py b/penny_dreadful_downloader/downloader/downloader.py
new file mode 100755
index 0000000..16d0dca
--- /dev/null
+++ b/penny_dreadful_downloader/downloader/downloader.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+
+import re
+import requests
+from bs4 import BeautifulSoup
+
+SEASON_FILES_DIRECTORY = "/home/viciouscirce/dox/project_mathemagicians/penny_dreadful/legality/legality_data/"
+
+def download_seasons_html():
+    page = requests.get("https://pennydreadfulmagic.com/seasons/")
+    return BeautifulSoup(page.text, "html.parser")
+
+def is_season(tag):
+    return tag.name == "section" and "class" in tag.attrs.keys() and "stats" in tag.attrs["class"]
+
+def get_season_number(tag):
+    header = tag.findChild("h2")
+    header_text = header.string
+    season_number = re.search("[0-9]{1,2}", header_text).group(0)
+    return season_number
+
+def get_season_cards_url(tag):
+    list_items = tag.findChildren("li")
+    legal_cards_item = list_items[8]
+    season_cards_url =  legal_cards_item.a.attrs["href"]
+    season_cards_url = season_cards_url.replace("https", "http")
+    return season_cards_url
+
+def get_seasons(soup):
+    season_tags = soup.find_all(is_season)
+
+    season_urls = {}
+
+    for season_tag in season_tags:
+        season_number = get_season_number(season_tag)
+        season_cards_url = get_season_cards_url(season_tag)
+        season_urls[season_number] = season_cards_url
+
+    return season_urls
+
+def download_season_lists(seasons):
+    for season_number, season_url in seasons.items():
+        cards_list = requests.get(season_url)
+        seasons[season_number] = cards_list.text
+
+    return seasons
+
+def write_season_lists(responses):
+    for season_number, cards_list in responses.items():
+        filename = SEASON_FILES_DIRECTORY + "season_" + season_number + "_legal_cards.txt"
+        with open(filename, "w") as cards_file:
+            cards_file.write(cards_list)
+
+def main():
+    soup = download_seasons_html()
+    seasons = get_seasons(soup)
+    lists = download_season_lists(seasons)
+    write_season_lists(lists)
+
+if __name__ == "__main__":
+    main()
diff --git a/penny_dreadful_downloader/downloader/runtest b/penny_dreadful_downloader/downloader/runtest
new file mode 100755
index 0000000..b74a08d
--- /dev/null
+++ b/penny_dreadful_downloader/downloader/runtest
@@ -0,0 +1,2 @@
+#!/bin/bash
+ls -1 downloader.py test_downloader.py | entr -cd python test_downloader.py
diff --git a/penny_dreadful_downloader/downloader/test_downloader.py b/penny_dreadful_downloader/downloader/test_downloader.py
new file mode 100644
index 0000000..7081902
--- /dev/null
+++ b/penny_dreadful_downloader/downloader/test_downloader.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+
+import unittest
+from unittest.mock import Mock, call, mock_open
+
+import requests
+from bs4 import BeautifulSoup
+
+import downloader
+
+def get_mock_cards_response(text):
+    mock_response = Mock()
+    mock_response.text = text
+    return mock_response
+
+class DownloaderTests(unittest.TestCase):
+    def test_download_seasons_html_calls_requests_get_with_correct_url(self):
+        requests.get = Mock()
+        requests.get.return_value.text = ''
+
+        downloader.download_seasons_html()
+
+        requests.get.assert_called_once_with("https://pennydreadfulmagic.com/seasons/")
+
+    def test_download_seasons_html_returns_soup_of_document(self):
+        mock_html = '<html></html>'
+        expected = BeautifulSoup(mock_html, "html.parser")
+
+        requests.get = Mock()
+        requests.get.return_value.text = mock_html
+
+        result = downloader.download_seasons_html()
+
+        self.assertEqual(expected, result)
+
+    def test_is_season_returns_true_when_tag_is_season(self):
+        tag = Mock()
+        tag.name = "section"
+        tag.attrs = {"class": "stats"}
+
+        is_season = downloader.is_season(tag)
+
+        self.assertEqual(is_season, True)
+
+    def test_is_season_returns_false_when_tag_name_is_not_div(self):
+        tag = Mock()
+        tag.name = "p"
+        tag.attrs = {"class": "stats"}
+
+        is_season = downloader.is_season(tag)
+
+        self.assertEqual(is_season, False)
+
+    def test_is_season_returns_false_when_tag_class_is_not_content_season(self):
+        tag = Mock()
+        tag.name = "section"
+        tag.attrs = {"class": "not-a-season"}
+
+        is_season = downloader.is_season(tag)
+
+        self.assertEqual(is_season, False)
+
+    def test_is_season_returns_false_when_tag_has_no_class(self):
+        tag = Mock()
+        tag.name = "section"
+        tag.attrs = {}
+
+        is_season = downloader.is_season(tag)
+
+        self.assertEqual(is_season, False)
+
+    def test_get_season_number_returns_correct_number(self):
+        expected_season_number = "29"
+        mock_soup = BeautifulSoup("<div><h2>Season " + expected_season_number + " (ONE)</h2></div>", "html.parser")
+        mock_tag = mock_soup.div
+
+        season_number = downloader.get_season_number(mock_tag)
+
+        self.assertEqual(season_number, expected_season_number)
+
+    def test_get_season_number_returns_correct_number_when_single_digit_season(self):
+        expected_season_number = "7"
+        mock_soup = BeautifulSoup("<div><h2>Season " + expected_season_number + " (RIX)</h2></div>", "html.parser")
+        mock_tag = mock_soup.div
+
+        season_number = downloader.get_season_number(mock_tag)
+
+        self.assertEqual(season_number, expected_season_number)
+
+    def test_get_season_cards_url_returns_correct_url_from_list(self):
+        expected_url = "TEST_SEASON_CARDS_URL"
+        mock_soup = BeautifulSoup('''<div class="content content-season"><section class="stats"><h2>Season 23 (VOW)</h2><p></p><ul><li></li><li></li><li></li><li></li><li></li><li></li><li></li><li></li><li><a href="''' + expected_url + '''">14,990 Legal Cards</a></li><li></li><li></li><li></li></ul></section></div>''', "html.parser")
+        mock_tag = mock_soup.div
+
+        season_url = downloader.get_season_cards_url(mock_tag)
+
+        self.assertEqual(season_url, expected_url)
+
+    def test_get_season_cards_url_returns_http_when_url_is_https(self):
+        expected_url = "http://pdmtgo.com/VOW_legal_cards.txt"
+        source_url = "https://pdmtgo.com/VOW_legal_cards.txt"
+        mock_soup = BeautifulSoup('''<div class="content content-season"><section class="stats"><h2>Season 23 (VOW)</h2><p></p><ul><li></li><li></li><li></li><li></li><li></li><li></li><li></li><li></li><li><a href="''' + source_url + '''">14,990 Legal Cards</a></li><li></li><li></li><li></li></ul></section></div>''', "html.parser")
+        mock_tag = mock_soup.div
+
+        season_url = downloader.get_season_cards_url(mock_tag)
+
+        self.assertEqual(season_url, expected_url)
+
+    def test_get_seasons_returns_dictionary_of_season_to_url(self):
+        expected_season_number = "23"
+        expected_url = "TEST_SEASON_CARDS_URL"
+        mock_soup = BeautifulSoup('''<html><body><main><div class="content content-seasons"><section class="stats"><h2>Season ''' + expected_season_number + ''' (VOW)</h2><p></p><ul><li></li><li></li><li></li><li></li><li></li><li></li><li></li><li></li><li><a href="''' + expected_url + '''">14,990 Legal Cards</a></li><li></li><li></li><li></li></ul></section></div></main></body></html>''', "html.parser")
+
+        seasons_list = downloader.get_seasons(mock_soup)
+
+        self.assertEqual(seasons_list, {expected_season_number: expected_url})
+
+    def test_download_season_lists_pulls_file_from_web(self):
+        expected_season_number = "23"
+        expected_url = "TEST_SEASON_CARDS_URL"
+
+        test_cards = "CARDS CARDS MORE CARDS"
+        requests.get = Mock(return_value=get_mock_cards_response(test_cards))
+
+        downloader.download_season_lists({expected_season_number: expected_url})
+
+        requests.get.assert_called_once_with(expected_url)            
+
+    def test_download_season_lists_gets_all_files(self):
+        test_url_1 = "SEASON_23_URL"
+        test_url_2 = "SEASON_42_URL"
+        test_seasons = {
+                "23": test_url_1,
+                "42": test_url_2}
+
+        test_cards = "CARDS CARDS MORE CARDS"
+        requests.get = Mock(return_value=get_mock_cards_response(test_cards))
+
+        downloader.download_season_lists(test_seasons)
+
+        requests.get.assert_has_calls([call(test_url_1), call(test_url_2)])
+
+    def test_write_season_lists_writes_response_text_to_disk(self):
+        test_cards = "CARDS CARDS MORE CARDS"
+        test_responses = {"23": test_cards}
+
+        open_mock = mock_open()
+        with unittest.mock.patch("builtins.open", open_mock):
+            downloader.write_season_lists(test_responses)
+
+        handle = open_mock()
+        handle.write.assert_called_with(test_cards)
+
+    def test_main_runs_program(self):
+        mock_soup = BeautifulSoup("<html></html>", "html.parser")
+        downloader.download_seasons_html = Mock(return_value=mock_soup)
+
+        mock_seasons = {"23": "URL"}
+        downloader.get_seasons = Mock(return_value=mock_seasons)
+
+        mock_lists = {"23": "CARDS UPON CARDS"}
+        downloader.download_season_lists = Mock(return_value=mock_lists)
+
+        # downloader.write_season_lists = Mock() # TODO: Find out why this causes the above test to fail
+
+        with unittest.mock.patch("builtins.open", mock_open()):
+            downloader.main()
+
+        downloader.download_seasons_html.assert_called_once()
+        downloader.get_seasons.assert_called_once_with(mock_soup)
+        downloader.download_season_lists.assert_called_once_with(mock_seasons)
+        # downloader.write_season_lists.assert_called_once_with(mock_lists)
+
+if __name__ == '__main__':
+    unittest.main()