diff --git a/README.md b/README.md index ad35b78b1e18c2345cb8f9f0688df4c763f7168a..703b12308e5d95f16d2ff27871e5e017a2a646f5 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,13 @@ # fa21-cs242-assignment-2 - +The project : fa21-cs242-assignment-2 consists of 3 different python +modules that include functions to take care of many of the core functionalities +of web scraping, transferring data to an external database, and also handling the usage +of a command line interface so that a user can export or import JSON data. +The Scraper Python Module handles the actual logic behind the scraping and parsing of website data +by using the inspect element function on the GoodReads.com website to find common attributes +that are shared between books and authors. The Database Python Module handles the transferring of data +that has been retrieved using the Scraper Module and utilizes MongoDB's cloud services to store data. +The CommandLine module is where the program sets up the command line interface for a user to interact with +and give valid inputs. The user should always supply at a minimum a starting book URL and the number of authors +and books to scrape. There is a also a test module that checks if helper functions used for the base functionality +are working as expected. Part 1 of this project has been completed as of 9/26/21. diff --git a/TestPlan.pdf b/TestPlan.pdf new file mode 100644 index 0000000000000000000000000000000000000000..84d50d71b085a95c894f168c296f5f65b26d2fe3 Binary files /dev/null and b/TestPlan.pdf differ diff --git a/Tests/robots.txt b/Tests/robots.txt new file mode 100644 index 0000000000000000000000000000000000000000..290553979bd17f61c154b8a5692b802f2d7a6adf --- /dev/null +++ b/Tests/robots.txt @@ -0,0 +1,1477 @@ +# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file +User-agent: * +Disallow: /about/team_member/ +Disallow: /admin +Disallow: /api +Disallow: /blog/list_rss +Disallow: /book/reviews/ +Disallow: /book_link/follow/ +Disallow: /buy_buttons/ +Disallow: /ebooks +Disallow: /event/show/ +Disallow: /home/index_rss +Disallow: /oggiPlayerLoader.htm +Disallow: /photo/group/ +Disallow: /quotes/list_rss +Disallow: /reader +Disallow: /review/list_rss +Disallow: /review/rate +Disallow: /shelf/user_shelves +Disallow: /story +Disallow: /tooltips +Disallow: /track +Disallow: /trivia/answer +Disallow: /user/updates_rss +Disallow: /book/show/52976360 +Disallow: /book/show/53024603 +Disallow: /book/show/52667524 +Disallow: /book/show/13634645 +Disallow: /book/show/53203379 +Disallow: /book/show/53472095 +Disallow: /book/show/53249757 +Disallow: /book/show/50692110 +Disallow: /book/show/53451162 +Disallow: /book/show/53005407 +Disallow: /book/show/50990510 +Disallow: /book/show/52558212 +Disallow: /book/show/52799562 +Disallow: /book/show/53179891 +Disallow: /book/show/53245386 +Disallow: /book/show/52719017 +Disallow: /book/show/53085988 +Disallow: /book/show/51038483 +Disallow: /book/show/51329187 +Disallow: /book/show/53280099 +Disallow: /book/show/52717468 +Disallow: /book/show/52976361 +Disallow: /book/show/53472951 +Disallow: /book/show/51108563 +Disallow: /book/show/53050699 +Disallow: /book/show/51918376 +Disallow: /book/show/51434342 +Disallow: /book/show/53201597 +Disallow: /book/show/53172541 +Disallow: /book/show/53232694 +Disallow: /book/show/53450079 +Disallow: /book/show/52427129 +Disallow: /book/show/52322894 +Disallow: /book/show/52047874 +Disallow: /book/show/52347423 +Disallow: /book/show/51204136 +Disallow: /book/show/52942181 +Disallow: /book/show/53186586 +Disallow: /book/show/51719335 +Disallow: /book/show/51698246 +Disallow: /book/show/52463373 +Disallow: /book/show/52788943 +Disallow: /book/show/51801441 +Disallow: /book/show/51213284 +Disallow: /book/show/51020287 +Disallow: /book/show/53089948 +Disallow: /book/show/53195440 +Disallow: /book/show/50631758 +Disallow: /book/show/51022002 +Disallow: /book/show/53191166 +Disallow: /book/show/50725198 +Disallow: /book/show/51201977 +Disallow: /book/show/53115195 +Disallow: /book/show/51797498 +Disallow: /book/show/53401351 +Disallow: /book/show/53041561 +Disallow: /book/show/52626808 +Disallow: /book/show/53065167 +Disallow: /book/show/51558207 +Disallow: /book/show/53309244 +Disallow: /book/show/51015019 +Disallow: /book/show/53253499 +Disallow: /book/show/53100533 +Disallow: /book/show/53188798 +Disallow: /book/show/53520556 +Disallow: /book/show/52330314 +Disallow: /book/show/53246535 +Disallow: /book/show/52409978 +Disallow: /book/show/51343168 +Disallow: /book/show/52850277 +Disallow: /book/show/52547736 +Disallow: /book/show/52316295 +Disallow: /book/show/51804919 +Disallow: /book/show/51074536 +Disallow: /book/show/53237780 +Disallow: /book/show/53403607 +Disallow: /book/show/51514388 +Disallow: /book/show/52342143 +Disallow: /book/show/52260125 +Disallow: /book/show/52980018 +Disallow: /book/show/53520178 +Disallow: /book/show/52321003 +Disallow: /book/show/50729730 +Disallow: /book/show/51088246 +Disallow: /book/show/52314567 +Disallow: /book/show/51665200 +Disallow: /book/show/52680803 +Disallow: /book/show/51974917 +Disallow: /book/show/51343069 +Disallow: /book/show/52168054 +Disallow: /book/show/53335220 +Disallow: /book/show/51288204 +Disallow: /book/show/53099359 +Disallow: /book/show/53121772 +Disallow: /book/show/53503520 +Disallow: /book/show/51480061 +Disallow: /book/show/51283207 +Disallow: /book/show/52909331 +Disallow: /book/show/50998173 +Disallow: /book/show/53209854 +Disallow: /book/show/53499858 +Disallow: /book/show/53520864 +Disallow: /book/show/52291307 +Disallow: /book/show/51912580 +Disallow: /book/show/53535568 +Disallow: /book/show/52891350 +Disallow: /book/show/53342241 +Disallow: /book/show/53049356 +Disallow: /book/show/53415676 +Disallow: /book/show/53498269 +Disallow: /book/show/50845468 +Disallow: /book/show/50646045 +Disallow: /book/show/53317061 +Disallow: /book/show/53199440 +Disallow: /book/show/52932479 +Disallow: /book/show/51489882 +Disallow: /book/show/51638665 +Disallow: /book/show/53413219 +Disallow: /book/show/51982695 +Disallow: /book/show/52207753 +Disallow: /book/show/53584054 +Disallow: /book/show/52586243 +Disallow: /book/show/53250220 +Disallow: /book/show/52356323 +Disallow: /book/show/53159602 +Disallow: /book/show/51628657 +Disallow: /book/show/52746896 +Disallow: /book/show/53342168 +Disallow: /book/show/50669879 +Disallow: /book/show/16764507 +Disallow: /book/show/53138175 +Disallow: /book/show/53316741 +Disallow: /book/show/53226126 +Disallow: /book/show/53266699 +Disallow: /book/show/53415793 +Disallow: /book/show/52728987 +Disallow: /book/show/53405678 +Disallow: /book/show/53404379 +Disallow: /book/show/52738243 +Disallow: /book/show/52862069 +Disallow: /book/show/53293383 +Disallow: /book/show/52384011 +Disallow: /book/show/53368848 +Disallow: /book/show/51474833 +Disallow: /book/show/51963393 +Disallow: /book/show/52125458 +Disallow: /book/show/52377651 +Disallow: /book/show/24214654 +Disallow: /book/show/51050037 +Disallow: /book/show/53265988 +Disallow: /book/show/53301691 +Disallow: /book/show/53056872 +Disallow: /book/show/53407827 +Disallow: /book/show/17863166 +Disallow: /book/show/52329175 +Disallow: /book/show/52894230 +Disallow: /book/show/53307347 +Disallow: /book/show/53185230 +Disallow: /book/show/53182783 +Disallow: /book/show/53347071 +Disallow: /book/show/53542879 +Disallow: /book/show/33830846 +Disallow: /book/show/50931526 +Disallow: /book/show/52047875 +Disallow: /book/show/53466741 +Disallow: /book/show/53186131 +Disallow: /book/show/51188143 +Disallow: /book/show/51440251 +Disallow: /book/show/51522609 +Disallow: /book/show/53318362 +Disallow: /book/show/53167505 +Disallow: /book/show/52591837 +Disallow: /book/show/51805269 +Disallow: /book/show/51273143 +Disallow: /book/show/50720331 +Disallow: /book/show/53340649 +Disallow: /book/show/53303241 +Disallow: /book/show/52971998 +Disallow: /book/show/51625769 +Disallow: /book/show/53104751 +Disallow: /book/show/53497400 +Disallow: /book/show/51092684 +Disallow: /book/show/51089096 +Disallow: /book/show/53353542 +Disallow: /book/show/53545733 +Disallow: /book/show/53183015 +Disallow: /book/show/53358630 +Disallow: /book/show/51601297 +Disallow: /book/show/53274500 +Disallow: /book/show/53183508 +Disallow: /book/show/51644240 +Disallow: /book/show/53295164 +Disallow: /book/show/52672548 +Disallow: /book/show/51790857 +Disallow: /book/show/53424952 +Disallow: /book/show/52862056 +Disallow: /book/show/51314687 +Disallow: /book/show/51628957 +Disallow: /book/show/52990698 +Disallow: /book/show/53465286 +Disallow: /book/show/53258347 +Disallow: /book/show/53446201 +Disallow: /book/show/50711760 +Disallow: /book/show/51811953 +Disallow: /book/show/51103126 +Disallow: /book/show/35488352 +Disallow: /book/show/53284551 +Disallow: /book/show/51798235 +Disallow: /book/show/51801829 +Disallow: /book/show/51097993 +Disallow: /book/show/51499340 +Disallow: /book/show/53446020 +Disallow: /book/show/51275027 +Disallow: /book/show/52665258 +Disallow: /book/show/53180522 +Disallow: /book/show/53408336 +Disallow: /book/show/51704606 +Disallow: /book/show/53569263 +Disallow: /book/show/53507643 +Disallow: /book/show/53530609 +Disallow: /book/show/52976178 +Disallow: /book/show/51492165 +Disallow: /book/show/52520444 +Disallow: /book/show/53162936 +Disallow: /book/show/53193649 +Disallow: /book/show/53287064 +Disallow: /book/show/52685330 +Disallow: /book/show/50993969 +Disallow: /book/show/52514726 +Disallow: /book/show/53520912 +Disallow: /book/show/53423561 +Disallow: /book/show/52662500 +Disallow: /book/show/52971482 +Disallow: /book/show/51468055 +Disallow: /book/show/50966995 +Disallow: /book/show/53289252 +Disallow: /book/show/52433587 +Disallow: /book/show/52731083 +Disallow: /book/show/53409792 +Disallow: /book/show/53244349 +Disallow: /book/show/51276381 +Disallow: /book/show/52296463 +Disallow: /book/show/52585794 +Disallow: /book/show/8640247 +Disallow: /book/show/50635412 +Disallow: /book/show/53348837 +Disallow: /book/show/52270874 +Disallow: /book/show/53522932 +Disallow: /book/show/53279909 +Disallow: /book/show/53348717 +Disallow: /book/show/53515927 +Disallow: /book/show/51001527 +Disallow: /book/show/53359292 +Disallow: /book/show/53141284 +Disallow: /book/show/52738252 +Disallow: /book/show/51113777 +Disallow: /book/show/53499994 +Disallow: /book/show/52349728 +Disallow: /book/show/51989048 +Disallow: /book/show/51178453 +Disallow: /book/show/52640822 +Disallow: /book/show/51588322 +Disallow: /book/show/53453323 +Disallow: /book/show/3758819 +Disallow: /book/show/51834786 +Disallow: /book/show/51160723 +Disallow: /book/show/53508500 +Disallow: /book/show/53511264 +Disallow: /book/show/51190957 +Disallow: /book/show/52415573 +Disallow: /book/show/53300423 +Disallow: /book/show/51023625 +Disallow: /book/show/52011682 +Disallow: /book/show/53179790 +Disallow: /book/show/53013223 +Disallow: /book/show/53663405 +Disallow: /book/show/51696925 +Disallow: /book/show/52074184 +Disallow: /book/show/53355616 +Disallow: /book/show/53340369 +Disallow: /book/show/51791640 +Disallow: /book/show/53309359 +Disallow: /book/show/52973230 +Disallow: /book/show/53519837 +Disallow: /book/show/53551065 +Disallow: /book/show/53405211 +Disallow: /book/show/50724086 +Disallow: /book/show/53125994 +Disallow: /book/show/53306688 +Disallow: /book/show/52727969 +Disallow: /book/show/53516666 +Disallow: /book/show/53064167 +Disallow: /book/show/53446005 +Disallow: /book/show/51493525 +Disallow: /book/show/50661501 +Disallow: /book/show/22590287 +Disallow: /book/show/51706849 +Disallow: /book/show/51872802 +Disallow: /book/show/53240092 +Disallow: /book/show/53400891 +Disallow: /book/show/51469847 +Disallow: /book/show/52650905 +Disallow: /book/show/51554773 +Disallow: /book/show/22397362 +Disallow: /book/show/34177971 +Disallow: /book/show/51284932 +Disallow: /book/show/42851955 +Disallow: /book/show/51033964 +Disallow: /book/show/52960574 +Disallow: /book/show/53127454 +Disallow: /book/show/53345857 +Disallow: /book/show/53516683 +Disallow: /book/show/51005209 +Disallow: /book/show/50725619 +Disallow: /book/show/51454502 +Disallow: /book/show/51953732 +Disallow: /book/show/52870566 +Disallow: /book/show/52888064 +Disallow: /book/show/51831518 +Disallow: /book/show/52408455 +Disallow: /book/show/19826709 +Disallow: /book/show/31803970 +Disallow: /book/show/51483933 +Disallow: /book/show/52608789 +Disallow: /book/show/6884041 +Disallow: /book/show/53350959 +Disallow: /book/show/53510525 +Disallow: /book/show/52520383 +Disallow: /book/show/53710501 +Disallow: /book/show/52520139 +Disallow: /book/show/51684336 +Disallow: /book/show/23457738 +Disallow: /book/show/53501665 +Disallow: /book/show/50653387 +Disallow: /book/show/50970546 +Disallow: /book/show/52660143 +Disallow: /book/show/53357162 +Disallow: /book/show/53202174 +Disallow: /book/show/51914177 +Disallow: /book/show/53528764 +Disallow: /book/show/53257489 +Disallow: /book/show/53402852 +Disallow: /book/show/53487336 +Disallow: /book/show/53497924 +Disallow: /book/show/52573941 +Disallow: /book/show/52911633 +Disallow: /book/show/53596972 +Disallow: /book/show/53666836 +Disallow: /book/show/53247969 +Disallow: /book/show/52809138 +Disallow: /book/show/53300043 +Disallow: /book/show/53354925 +Disallow: /book/show/51684335 +Disallow: /book/show/53480644 +Disallow: /book/show/53071448 +Disallow: /book/show/53534416 +Disallow: /book/show/51566603 +Disallow: /book/show/53187865 +Disallow: /book/show/53343670 +Disallow: /book/show/53464282 +Disallow: /book/show/53495247 +Disallow: /book/show/53661407 +Disallow: /book/show/34287234 +Disallow: /book/show/53412080 +Disallow: /book/show/53507024 +Disallow: /book/show/50716965 +Disallow: /book/show/51799054 +Disallow: /book/show/51054623 +Disallow: /book/show/53528191 +Disallow: /book/show/53455498 +Disallow: /book/show/53108139 +Disallow: /book/show/50716137 +Disallow: /book/show/53126002 +Disallow: /book/show/52834215 +Disallow: /book/show/53549683 +Disallow: /book/show/52917857 +Disallow: /book/show/34289082 +Disallow: /book/show/52929150 +Disallow: /book/show/52520114 +Disallow: /book/show/53363289 +Disallow: /book/show/16831051 +Disallow: /book/show/53363273 +Disallow: /book/show/51982726 +Disallow: /book/show/50692675 +Disallow: /book/show/34189914 +Disallow: /book/show/53514379 +Disallow: /book/show/53469492 +Disallow: /book/show/51684339 +Disallow: /book/show/53252344 +Disallow: /book/show/53231226 +Disallow: /book/show/53362469 +Disallow: /book/show/51278589 +Disallow: /book/show/53596137 +Disallow: /book/show/53543107 +Disallow: /book/show/53655569 +Disallow: /book/show/53534736 +Disallow: /book/show/51982371 +Disallow: /book/show/51801380 +Disallow: /book/show/53348468 +Disallow: /book/show/53335074 +Disallow: /book/show/5103377 +Disallow: /book/show/52627506 +Disallow: /book/show/51174267 +Disallow: /book/show/53569995 +Disallow: /book/show/52549736 +Disallow: /book/show/52721062 +Disallow: /book/show/51113343 +Disallow: /book/show/51441556 +Disallow: /book/show/53388694 +Disallow: /book/show/52997248 +Disallow: /book/show/52738192 +Disallow: /book/show/53179145 +Disallow: /book/show/53548349 +Disallow: /book/show/53495994 +Disallow: /book/show/52217335 +Disallow: /book/show/52563624 +Disallow: /book/show/52280158 +Disallow: /book/show/52662423 +Disallow: /book/show/53624050 +Disallow: /book/show/50866107 +Disallow: /book/show/53621129 +Disallow: /book/show/53694323 +Disallow: /book/show/52518611 +Disallow: /book/show/52662465 +Disallow: /book/show/53127678 +Disallow: /book/show/53683095 +Disallow: /book/show/53264384 +Disallow: /book/show/53479530 +Disallow: /book/show/53542891 +Disallow: /book/show/52927050 +Disallow: /book/show/51234907 +Disallow: /book/show/53530193 +Disallow: /book/show/53437151 +Disallow: /book/show/53292784 +Disallow: /book/show/53447330 +Disallow: /book/show/53575691 +Disallow: /book/show/53240761 +Disallow: /book/show/53387536 +Disallow: /book/show/53362413 +Disallow: /book/show/52833865 +Disallow: /book/show/52673740 +Disallow: /book/show/50714213 +Disallow: /book/show/53343505 +Disallow: /book/show/52520410 +Disallow: /book/show/51686013 +Disallow: /book/show/53564003 +Disallow: /book/show/53647288 +Disallow: /book/show/53359022 +Disallow: /book/show/53717436 +Disallow: /book/show/52867708 +Disallow: /book/show/21033144 +Disallow: /book/show/53558224 +Disallow: /book/show/52394964 +Disallow: /book/show/52990030 +Disallow: /book/show/50661179 +Disallow: /book/show/52662479 +Disallow: /book/show/50716867 +Disallow: /book/show/53637867 +Disallow: /book/show/53363183 +Disallow: /book/show/53689846 +Disallow: /book/show/51982898 +Disallow: /book/show/53632765 +Disallow: /book/show/53183122 +Disallow: /book/show/53489479 +Disallow: /book/show/53606563 +Disallow: /book/show/52541834 +Disallow: /book/show/53533690 +Disallow: /book/show/51453412 +Disallow: /book/show/52658434 +Disallow: /book/show/53126985 +Disallow: /book/show/53623342 +Disallow: /book/show/53702361 +Disallow: /book/show/18567421 +Disallow: /book/show/21512167 +Disallow: /book/show/52269497 +Disallow: /book/show/53127504 +Disallow: /book/show/53256867 +Disallow: /book/show/53519960 +Disallow: /book/show/51626172 +Disallow: /book/show/53458133 +Disallow: /book/show/52894580 +Disallow: /book/show/53473989 +Disallow: /book/show/53451753 +Disallow: /book/show/51982929 +Disallow: /book/show/53400991 +Disallow: /book/show/53573546 +Disallow: /book/show/52520288 +Disallow: /book/show/52770593 +Disallow: /book/show/52821638 +Disallow: /book/show/53166633 +Disallow: /book/show/53583454 +Disallow: /book/show/53349389 +Disallow: /book/show/51080241 +Disallow: /book/show/52603385 +Disallow: /book/show/43502994 +Disallow: /book/show/53698167 +Disallow: /book/show/52822368 +Disallow: /book/show/53296863 +Disallow: /book/show/53397732 +Disallow: /book/show/52217367 +Disallow: /book/show/53613999 +Disallow: /book/show/52540278 +Disallow: /book/show/51353738 +Disallow: /book/show/51454279 +Disallow: /book/show/34254207 +Disallow: /book/show/53356908 +Disallow: /book/show/51953078 +Disallow: /book/show/53363881 +Disallow: /book/show/53260495 +Disallow: /book/show/53479546 +Disallow: /book/show/34177972 +Disallow: /book/show/53457345 +Disallow: /book/show/53712282 +Disallow: /book/show/10163692 +Disallow: /book/show/53155977 +Disallow: /book/show/51119709 +Disallow: /book/show/52454841 +Disallow: /book/show/53529360 +Disallow: /book/show/52834048 +Disallow: /book/show/52360488 +Disallow: /book/show/53241635 +Disallow: /book/show/34287175 +Disallow: /book/show/52208128 +Disallow: /book/show/53661367 +Disallow: /book/show/51619314 +Disallow: /book/show/53453282 +Disallow: /book/show/52520170 +Disallow: /book/show/53409817 +Disallow: /book/show/52586350 +Disallow: /book/show/53467058 +Disallow: /book/show/53502475 +Disallow: /book/show/53452148 +Disallow: /book/show/52520225 +Disallow: /book/show/53642693 +Disallow: /book/show/53509357 +Disallow: /book/show/53162871 +Disallow: /book/show/52729393 +Disallow: /book/show/53263125 +Disallow: /book/show/51687282 +Disallow: /book/show/51039955 +Disallow: /book/show/53429581 +Disallow: /book/show/53363216 +Disallow: /book/show/53634016 +Disallow: /book/show/53596040 +Disallow: /book/show/53416495 +Disallow: /book/show/53248749 +Disallow: /book/show/51575337 +Disallow: /book/show/53072919 +Disallow: /book/show/30257708 +Disallow: /book/show/52546902 +Disallow: /book/show/53553537 +Disallow: /book/show/53511548 +Disallow: /book/show/53025880 +Disallow: /book/show/52746939 +Disallow: /book/show/52833965 +Disallow: /book/show/53356395 +Disallow: /book/show/52279016 +Disallow: /book/show/53583343 +Disallow: /book/show/53457138 +Disallow: /book/show/22644519 +Disallow: /book/show/53468028 +Disallow: /book/show/4727315 +Disallow: /book/show/53535689 +Disallow: /book/show/16973102 +Disallow: /book/show/53206948 +Disallow: /book/show/53458852 +Disallow: /book/show/53521460 +Disallow: /book/show/156958 +Disallow: /book/show/53354732 +Disallow: /book/show/53438075 +Disallow: /book/show/53483912 +Disallow: /book/show/53516787 +Disallow: /book/show/53362446 +Disallow: /book/show/53666861 +Disallow: /book/show/50855939 +Disallow: /book/show/53634225 +Disallow: /book/show/52833714 +Disallow: /book/show/51709367 +Disallow: /book/show/53410750 +Disallow: /book/show/52833693 +Disallow: /book/show/52093451 +Disallow: /book/show/51161117 +Disallow: /book/show/53427949 +Disallow: /book/show/53344928 +Disallow: /book/show/53569775 +Disallow: /book/show/53321003 +Disallow: /book/show/52453381 +Disallow: /book/show/53511850 +Disallow: /book/show/24216868 +Disallow: /book/show/51819709 +Disallow: /book/show/53491553 +Disallow: /book/show/53267635 +Disallow: /book/show/53562990 +Disallow: /book/show/53257843 +Disallow: /book/show/53525230 +Disallow: /book/show/53556607 +Disallow: /book/show/18672665 +Disallow: /book/show/53439592 +Disallow: /book/show/51440146 +Disallow: /book/show/53297627 +Disallow: /book/show/53218600 +Disallow: /book/show/53334665 +Disallow: /book/show/50910447 +Disallow: /book/show/53529081 +Disallow: /book/show/53273798 +Disallow: /book/show/51350549 +Disallow: /book/show/53596165 +Disallow: /book/show/53218793 +Disallow: /book/show/53380034 +Disallow: /book/show/52978676 +Disallow: /book/show/53517673 +Disallow: /book/show/53583492 +Disallow: /book/show/51638701 +Disallow: /book/show/51982288 +Disallow: /book/show/53333526 +Disallow: /book/show/52450986 +Disallow: /book/show/53193638 +Disallow: /book/show/53516922 +Disallow: /book/show/51783124 +Disallow: /book/show/53419014 +Disallow: /book/show/52716552 +Disallow: /book/show/52384539 +Disallow: /book/show/53426666 +Disallow: /book/show/52682789 +Disallow: /book/show/51561112 +Disallow: /book/show/51610697 +Disallow: /book/show/53521015 +Disallow: /book/show/53555986 +Disallow: /book/show/53274383 +Disallow: /book/show/52821639 +Disallow: /book/show/53110914 +Disallow: /book/show/53197697 +Disallow: /book/show/52718730 +Disallow: /book/show/52662472 +Disallow: /book/show/53634619 +Disallow: /book/show/53608347 +Disallow: /book/show/14717321 +Disallow: /book/show/53215728 +Disallow: /book/show/52412155 +Disallow: /book/show/34169240 +Disallow: /book/show/34168850 +Disallow: /book/show/53411235 +Disallow: /book/show/27015538 +Disallow: /book/show/53484085 +Disallow: /book/show/53711283 +Disallow: /book/show/52817135 +Disallow: /book/show/51710375 +Disallow: /book/show/53181456 +Disallow: /book/show/53404064 +Disallow: /book/show/53490874 +Disallow: /book/show/34287250 +Disallow: /book/show/53555284 +Disallow: /book/show/34254456 +Disallow: /book/show/53438469 +Disallow: /book/show/23147490 +Disallow: /book/show/53233661 +Disallow: /book/show/53658854 +Disallow: /book/show/53286071 +Disallow: /book/show/53376964 +Disallow: /book/show/53441987 +Disallow: /book/show/53297686 +Disallow: /book/show/53612852 +Disallow: /book/show/53600376 +Disallow: /book/show/27978878 +Disallow: /book/show/34130940 +Disallow: /book/show/53698184 +Disallow: /book/show/53163305 +Disallow: /book/show/50730982 +Disallow: /book/show/53579248 +Disallow: /book/show/34173072 +Disallow: /book/show/34194331 +Disallow: /book/show/53464444 +Disallow: /book/show/53584101 +Disallow: /book/show/53452383 +Disallow: /book/show/53627149 +Disallow: /book/show/53561412 +Disallow: /book/show/53438362 +Disallow: /book/show/53458293 +Disallow: /book/show/53698216 +Disallow: /book/show/53660340 +Disallow: /book/show/53363944 +Disallow: /book/show/51437916 +Disallow: /book/show/23147628 +Disallow: /book/show/53351682 +Disallow: /book/show/53652041 +Disallow: /book/show/34573991 +Disallow: /book/show/51408657 +Disallow: /book/show/53611459 +Disallow: /book/show/53583361 +Disallow: /book/show/53615968 +Disallow: /book/show/53190598 +Disallow: /book/show/53687790 +Disallow: /book/show/52536248 +Disallow: /book/show/53620974 +Disallow: /book/show/52792959 +Disallow: /book/show/53677222 +Disallow: /book/show/34158124 +Disallow: /book/show/53545749 +Disallow: /book/show/52703458 +Disallow: /book/show/53399542 +Disallow: /book/show/52536246 +Disallow: /book/show/53363945 +Disallow: /book/show/51203915 +Disallow: /book/show/34279564 +Disallow: /book/show/34131467 +Disallow: /book/show/52662470 +Disallow: /book/show/53344745 +Disallow: /book/show/34131290 +Disallow: /book/show/53544979 +Disallow: /book/show/51884359 +Disallow: /book/show/33705838 +Disallow: /book/show/53551387 +Disallow: /book/show/53672726 +Disallow: /book/show/51664121 +Disallow: /book/show/53716238 +Disallow: /book/show/51668879 +Disallow: /book/show/53517236 +Disallow: /book/show/53557366 +Disallow: /book/show/53134919 +Disallow: /book/show/52662443 +Disallow: /book/show/53568703 +Disallow: /book/show/52179692 +Disallow: /book/show/50762294 +Disallow: /book/show/53214825 +Disallow: /book/show/51282946 +Disallow: /book/show/52854182 +Disallow: /book/show/52662455 +Disallow: /book/show/53588635 +Disallow: /book/show/51425410 +Disallow: /book/show/23458058 +Disallow: /book/show/53665859 +Disallow: /book/show/53505483 +Disallow: /book/show/53180115 +Disallow: /book/show/51626101 +Disallow: /book/show/34296467 +Disallow: /book/show/53455262 +Disallow: /book/show/53721548 +Disallow: /book/show/39971514 +Disallow: /book/show/34288128 +Disallow: /book/show/53708337 +Disallow: /book/show/51415009 +Disallow: /book/show/53509151 +Disallow: /book/show/52658354 +Disallow: /book/show/53632749 +Disallow: /book/show/53601627 +Disallow: /book/show/18566580 +Disallow: /book/show/53301073 +Disallow: /book/show/53471283 +Disallow: /book/show/51204955 +Disallow: /book/show/53163304 +Disallow: /book/show/53243683 +Disallow: /book/show/34190006 +Disallow: /book/show/53343583 +Disallow: /book/show/34173142 +Disallow: /book/show/52563679 +Disallow: /book/show/53686425 +Disallow: /book/show/52563523 +Disallow: /book/show/53442629 +Disallow: /book/show/53643961 +Disallow: /book/show/53478962 +Disallow: /book/show/51253155 +Disallow: /book/show/53625537 +Disallow: /book/show/53625467 +Disallow: /book/show/53426459 +Disallow: /book/show/53665703 +Disallow: /book/show/53489262 +Disallow: /book/show/50750102 +Disallow: /book/show/53589341 +Disallow: /book/show/53413555 +Disallow: /book/show/53312217 +Disallow: /book/show/53371770 +Disallow: /book/show/53678319 +Disallow: /book/show/53189964 +Disallow: /book/show/53288769 +Disallow: /book/show/53459470 +Disallow: /book/show/53481162 +Disallow: /book/show/53274062 +Disallow: /book/show/52962635 +Disallow: /book/show/53617336 +Disallow: /book/show/53512133 +Disallow: /book/show/53521833 +Disallow: /book/show/53574523 +Disallow: /book/show/53480848 +Disallow: /book/show/53531107 +Disallow: /book/show/53711687 +Disallow: /book/show/34255048 +Disallow: /book/show/53505624 +Disallow: /book/show/53595016 +Disallow: /book/show/53243682 +Disallow: /book/show/52731420 +Disallow: /book/show/34171407 +Disallow: /book/show/53591706 +Disallow: /book/show/18567569 +Disallow: /book/show/53500182 +Disallow: /book/show/53549795 +Disallow: /book/show/53578606 +Disallow: /book/show/53458180 +Disallow: /book/show/52611379 +Disallow: /book/show/52854005 +Disallow: /book/show/53644828 +Disallow: /book/show/50125995 +Disallow: /book/show/52563920 +Disallow: /book/show/53354230 +Disallow: /book/show/53520195 +Disallow: /book/show/53514157 +Disallow: /book/show/53725539 +Disallow: /book/show/53225550 +Disallow: /book/show/51430795 +Disallow: /book/show/52662391 +Disallow: /book/show/53699439 +Disallow: /book/show/53263101 +Disallow: /book/show/53582435 +Disallow: /book/show/53320021 +Disallow: /book/show/34168755 +Disallow: /book/show/53614059 +Disallow: /book/show/53242804 +Disallow: /book/show/53513213 +Disallow: /book/show/53577149 +Disallow: /book/show/52692936 +Disallow: /book/show/51001515 +Disallow: /book/show/53351110 +Disallow: /book/show/51624433 +Disallow: /book/show/33706037 +Disallow: /book/show/20864678 +Disallow: /book/show/53491462 +Disallow: /book/show/53634451 +Disallow: /book/show/52608613 +Disallow: /book/show/34287174 +Disallow: /book/show/53359021 +Disallow: /book/show/53441655 +Disallow: /book/show/21744548 +Disallow: /book/show/53591266 +Disallow: /book/show/53550631 +Disallow: /book/show/51982951 +Disallow: /book/show/53363954 +Disallow: /book/show/51553711 +Disallow: /book/show/53640931 +Disallow: /book/show/53452503 +Disallow: /book/show/34169062 +Disallow: /book/show/52827771 +Disallow: /book/show/53211206 +Disallow: /book/show/53440120 +Disallow: /book/show/53516976 +Disallow: /book/show/51876430 +Disallow: /book/show/52662451 +Disallow: /book/show/51427091 +Disallow: /book/show/52608764 +Disallow: /book/show/51480501 +Disallow: /book/show/53705528 +Disallow: /book/show/53430451 +Disallow: /book/show/53210542 +Disallow: /book/show/53409517 +Disallow: /book/show/52794177 +Disallow: /book/show/53653230 +Disallow: /book/show/53578498 +Disallow: /book/show/53566096 +Disallow: /book/show/34288159 +Disallow: /book/show/51264756 +Disallow: /book/show/53643048 +Disallow: /book/show/34255468 +Disallow: /book/show/53457479 +Disallow: /book/show/53561736 +Disallow: /book/show/53528849 +Disallow: /book/show/53493057 +Disallow: /book/show/53585835 +Disallow: /book/show/53254160 +Disallow: /book/show/53610275 +Disallow: /book/show/53476328 +Disallow: /book/show/53721586 +Disallow: /book/show/2869329 +Disallow: /book/show/53605915 +Disallow: /book/show/52559471 +Disallow: /book/show/53363955 +Disallow: /book/show/53725875 +Disallow: /book/show/53372229 +Disallow: /book/show/53421661 +Disallow: /book/show/53512115 +Disallow: /book/show/53603688 +Disallow: /book/show/26500885 +Disallow: /book/show/53232699 +Disallow: /book/show/51798613 +Disallow: /book/show/52563608 +Disallow: /book/show/53314238 +Disallow: /book/show/53647096 +Disallow: /book/show/52662450 +Disallow: /book/show/53490860 +Disallow: /book/show/53624309 +Disallow: /book/show/50810529 +Disallow: /book/show/34194621 +Disallow: /book/show/53505711 +Disallow: /book/show/53595778 +Disallow: /book/show/53251296 +Disallow: /book/show/51455155 +Disallow: /book/show/53684689 +Disallow: /book/show/53502493 +Disallow: /book/show/53658443 +Disallow: /book/show/33472884 +Disallow: /book/show/53425750 +Disallow: /book/show/53559487 +Disallow: /book/show/52563558 +Disallow: /book/show/53494580 +Disallow: /book/show/51701884 +Disallow: /book/show/10163691 +Disallow: /book/show/53436070 +Disallow: /book/show/53648413 +Disallow: /book/show/52518753 +Disallow: /book/show/53218799 +Disallow: /book/show/53349610 +Disallow: /book/show/53363952 +Disallow: /book/show/53521045 +Disallow: /book/show/52827684 +Disallow: /book/show/52563736 +Disallow: /book/show/34173078 +Disallow: /book/show/53481245 +Disallow: /book/show/53621497 +Disallow: /book/show/53603576 +Disallow: /book/show/53547888 +Disallow: /book/show/53479387 +Disallow: /book/show/53200659 +Disallow: /book/show/52662467 +Disallow: /book/show/53476685 +Disallow: /book/show/34287241 +Disallow: /book/show/53148601 +Disallow: /book/show/53335033 +Disallow: /book/show/53467867 +Disallow: /book/show/51440032 +Disallow: /book/show/53598148 +Disallow: /book/show/48945447 +Disallow: /book/show/53541044 +Disallow: /book/show/53598217 +Disallow: /book/show/53614210 +Disallow: /book/show/53625617 +Disallow: /book/show/51139248 +Disallow: /book/show/22644355 +Disallow: /book/show/53009327 +Disallow: /book/show/53592321 +Disallow: /book/show/53651753 +Disallow: /book/show/34168533 +Disallow: /book/show/53644823 +Disallow: /book/show/53681453 +Disallow: /book/show/53363938 +Disallow: /book/show/50660666 +Disallow: /book/show/52563598 +Disallow: /book/show/52893797 +Disallow: /book/show/53481160 +Disallow: /book/show/53300747 +Disallow: /book/show/53210273 +Disallow: /book/show/53485762 +Disallow: /book/show/53363942 +Disallow: /book/show/34289083 +Disallow: /book/show/53216010 +Disallow: /book/show/53615783 +Disallow: /book/show/32440801 +Disallow: /book/show/53436076 +Disallow: /book/show/51601223 +Disallow: /book/show/51402794 +Disallow: /book/show/53303249 +Disallow: /book/show/51085723 +Disallow: /book/show/53667647 +Disallow: /book/show/53626450 +Disallow: /book/show/52744522 +Disallow: /book/show/53493112 +Disallow: /book/show/53636037 +Disallow: /book/show/37865399 +Disallow: /book/show/53488131 +Disallow: /book/show/53632421 +Disallow: /book/show/53611177 +Disallow: /book/show/53707150 +Disallow: /book/show/53591403 +Disallow: /book/show/50983627 +Disallow: /book/show/52693334 +Disallow: /book/show/51797486 +Disallow: /book/show/53262613 +Disallow: /book/show/53709892 +Disallow: /book/show/53481299 +Disallow: /book/show/53363949 +Disallow: /book/show/51425575 +Disallow: /book/show/53330209 +Disallow: /book/show/53557697 +Disallow: /book/show/53254152 +Disallow: /book/show/51266729 +Disallow: /book/show/53635851 +Disallow: /book/show/34290056 +Disallow: /book/show/51441648 +Disallow: /book/show/33471804 +Disallow: /book/show/53478961 +Disallow: /book/show/51093786 +Disallow: /book/show/51443061 +Disallow: /book/show/53363940 +Disallow: /book/show/53247963 +Disallow: /book/show/53425912 +Disallow: /book/show/52662442 +Disallow: /book/show/51003725 +Disallow: /book/show/53192943 +Disallow: /book/show/52510333 +Disallow: /book/show/53435462 +Disallow: /book/show/34275723 +Disallow: /book/show/34288011 +Disallow: /book/show/53493810 +Disallow: /book/show/51438503 +Disallow: /book/show/51624282 +Disallow: /book/show/53637975 +Disallow: /book/show/52518649 +Disallow: /book/show/34288129 +Disallow: /book/show/53231937 +Disallow: /book/show/51000258 +Disallow: /book/show/53238309 +Disallow: /book/show/51609589 +Disallow: /book/show/52536243 +Disallow: /book/show/53714460 +Disallow: /book/show/53589482 +Disallow: /book/show/53687262 +Disallow: /book/show/53386159 +Disallow: /book/show/34287242 +Disallow: /book/show/51684337 +Disallow: /book/show/53643894 +Disallow: /book/show/52165506 +Disallow: /book/show/51818397 +Disallow: /book/show/51217710 +Disallow: /book/show/52608537 +Disallow: /book/show/53706624 +Disallow: /book/show/53473998 +Disallow: /book/show/53612816 +Disallow: /book/show/53487321 +Disallow: /book/show/53583417 +Disallow: /book/show/53363943 +Disallow: /book/show/53538170 +Disallow: /book/show/53638386 +Disallow: /book/show/51623136 +Disallow: /book/show/53348282 +Disallow: /book/show/53273635 +Disallow: /book/show/18566928 +Disallow: /book/show/53590565 +Disallow: /book/show/53459991 +Disallow: /book/show/53612098 +Disallow: /book/show/52608800 +Disallow: /book/show/53456398 +Disallow: /book/show/53026538 +Disallow: /book/show/51437903 +Disallow: /book/show/5299838 +Disallow: /book/show/53576629 +Disallow: /book/show/53504996 +Disallow: /book/show/53363948 +Disallow: /book/show/53705849 +Disallow: /book/show/52662400 +Disallow: /book/show/53524156 +Disallow: /book/show/53510084 +Disallow: /book/show/53442242 +Disallow: /book/show/23425930 +Disallow: /book/show/52662392 +Disallow: /book/show/52073888 +Disallow: /book/show/53591274 +Disallow: /book/show/53218800 +Disallow: /book/show/53478973 +Disallow: /book/show/53539280 +Disallow: /book/show/53476741 +Disallow: /book/show/9929315 +Disallow: /book/show/4408928 +Disallow: /book/show/53532288 +Disallow: /book/show/3278203 +Disallow: /book/show/53339198 +Disallow: /book/show/53629107 +Disallow: /book/show/52794181 +Disallow: /book/show/51439957 +Disallow: /book/show/50788212 +Disallow: /book/show/53564244 +Disallow: /book/show/53178369 +Disallow: /book/show/34287236 +Disallow: /book/show/52885739 +Disallow: /book/show/51137860 +Disallow: /book/show/53546375 +Disallow: /book/show/52662240 +Disallow: /book/show/53258377 +Disallow: /book/show/34287235 +Disallow: /book/show/11751387 +Disallow: /book/show/52518878 +Disallow: /book/show/53723579 +Disallow: /book/show/53595691 +Disallow: /book/show/52662459 +Disallow: /book/show/53590925 +Disallow: /book/show/53425762 +Disallow: /book/show/52242114 +Disallow: /book/show/51145919 +Disallow: /book/show/53458204 +Disallow: /book/show/33317665 +Disallow: /book/show/53502109 +Disallow: /book/show/53705525 +Disallow: /book/show/53560403 +Disallow: /book/show/52662462 +Disallow: /book/show/53605896 +Disallow: /book/show/53530009 +Disallow: /book/show/53413536 +Disallow: /book/show/53589973 +Disallow: /book/show/53218794 +Disallow: /book/show/10428516 +Disallow: /book/show/53637728 +Disallow: /book/show/27185763 +Disallow: /book/show/18767837 +Disallow: /book/show/53622208 +Disallow: /book/show/34194182 +Disallow: /book/show/53303373 +Disallow: /book/show/33476020 +Disallow: /book/show/50902332 +Disallow: /book/show/34287237 +Disallow: /book/show/53435458 +Disallow: /book/show/53663342 +Disallow: /book/show/50835664 +Disallow: /book/show/53681219 +Disallow: /book/show/53427858 +Disallow: /book/show/52820594 +Disallow: /book/show/53524128 +Disallow: /book/show/53551099 +Disallow: /book/show/53481282 +Disallow: /book/show/51726180 +Disallow: /book/show/53587688 +Disallow: /book/show/53525698 +Disallow: /book/show/33420292 +Disallow: /book/show/52025670 +Disallow: /book/show/53654743 +Disallow: /book/show/34290057 +Disallow: /book/show/53437816 +Disallow: /book/show/51620264 +Disallow: /book/show/29068443 +Disallow: /book/show/52420665 +Disallow: /book/show/53558943 +Disallow: /book/show/52608760 +Disallow: /book/show/53466332 +Disallow: /book/show/52829626 +Disallow: /book/show/53654454 +Disallow: /book/show/52991067 +Disallow: /book/show/53623100 +Disallow: /book/show/53609286 +Disallow: /book/show/51440070 +Disallow: /book/show/53634417 +Disallow: /book/show/52608874 +Disallow: /book/show/53706090 +Disallow: /book/show/53449731 +Disallow: /book/show/53026527 +Disallow: /book/show/53530537 +Disallow: /book/show/53512748 +Disallow: /book/show/34190260 +Disallow: /book/show/53632827 +Disallow: /book/show/53671204 +Disallow: /book/show/53353350 +Disallow: /book/show/51804457 +Disallow: /book/show/53678290 +Disallow: /book/show/53623372 +Disallow: /book/show/53342984 +Disallow: /book/show/53567267 +Disallow: /book/show/53702360 +Disallow: /book/show/4613791 +Disallow: /book/show/53572156 +Disallow: /book/show/16761441 +Disallow: /book/show/53404008 +Disallow: /book/show/53466613 +Disallow: /book/show/53625094 +Disallow: /book/show/20241987 +Disallow: /book/show/53483173 +Disallow: /book/show/53467753 +Disallow: /book/show/51423723 +Disallow: /book/show/53214826 +Disallow: /book/show/52850276 +Disallow: /book/show/34279682 +Disallow: /book/show/53480556 +Disallow: /book/show/53673284 +Disallow: /book/show/53701884 +Disallow: /book/show/53675650 +Disallow: /book/show/50672021 +Disallow: /book/show/53541199 +Disallow: /book/show/53721666 +Disallow: /book/show/53592270 +Disallow: /book/show/53576635 +Disallow: /book/show/53264012 +Disallow: /book/show/53493156 +Disallow: /book/show/53554346 +Disallow: /book/show/53606697 +Disallow: /book/show/53377204 +Disallow: /book/show/52071914 +Disallow: /book/show/52662440 +Disallow: /book/show/34135341 +Disallow: /book/show/53504764 +Disallow: /book/show/53363951 +Disallow: /book/show/53363946 +Disallow: /book/show/34173071 +Disallow: /book/show/53728203 +Disallow: /book/show/52354633 +Disallow: /book/show/53524106 +Disallow: /book/show/53210709 +Disallow: /book/show/53527488 +Disallow: /book/show/53706547 +Disallow: /book/show/53654463 +Disallow: /book/show/52827621 +Disallow: /book/show/53347742 +Disallow: /book/show/53459063 +Disallow: /book/show/51589417 +Disallow: /book/show/51024021 +Disallow: /book/show/53358869 +Disallow: /book/show/53481241 +Disallow: /book/show/51684331 +Disallow: /book/show/53456425 +Disallow: /book/show/11209920 +Disallow: /book/show/34254334 +Disallow: /book/show/53054350 +Disallow: /book/show/52662226 +Disallow: /book/show/34287251 +Disallow: /book/show/52697753 +Disallow: /book/show/53624859 +Disallow: /book/show/53552002 +Disallow: /book/show/51428157 +Disallow: /book/show/53664840 +Disallow: /book/show/53654455 +Disallow: /book/show/53595232 +Disallow: /book/show/52662404 +Disallow: /book/show/53451228 +Disallow: /book/show/53585296 +Disallow: /book/show/34254307 +Disallow: /book/show/53625544 +Disallow: /book/show/53180116 +Disallow: /book/show/53653831 +Disallow: /book/show/53664518 +Disallow: /book/show/51441722 +Disallow: /book/show/53298940 +Disallow: /book/show/53697675 +Disallow: /book/show/53612095 +Disallow: /book/show/52069932 +Disallow: /book/show/53557487 +Disallow: /book/show/52662449 +Disallow: /book/show/52807966 +Disallow: /book/show/53512143 +Disallow: /book/show/53517365 +Disallow: /book/show/53512687 +Disallow: /book/show/53300437 +Disallow: /book/show/52165543 +Disallow: /book/show/53689775 +Disallow: /book/show/53605879 +Disallow: /book/show/51313103 +Disallow: /book/show/51145940 +Disallow: /book/show/53568704 +Disallow: /book/show/52572629 +Disallow: /book/show/53575153 +Disallow: /book/show/53694185 +Disallow: /book/show/53449694 +Disallow: /book/show/53566390 +Disallow: /book/show/43999010 +Disallow: /book/show/53707826 +Disallow: /book/show/53578754 +Disallow: /book/show/53218798 +Disallow: /book/show/50928790 +Disallow: /book/show/53180114 +Disallow: /book/show/53371772 +Disallow: /book/show/53679298 +Disallow: /book/show/51982978 +Disallow: /book/show/53516279 +Disallow: /book/show/34275904 +Disallow: /book/show/19734747 +Disallow: /book/show/34173992 +Disallow: /book/show/34254084 +Disallow: /book/show/53321002 +Disallow: /book/show/51012434 +Disallow: /book/show/53587785 +Disallow: /book/show/19985126 +Disallow: /book/show/2729024 +Disallow: /book/show/53643951 +Disallow: /book/show/52984991 +Disallow: /book/show/53716878 +Disallow: /book/show/51982425 +Disallow: /book/show/51446394 +Disallow: /book/show/53720829 +Disallow: /book/show/51623504 +Disallow: /book/show/53398392 +Disallow: /book/show/53431921 +Disallow: /book/show/53334341 +Disallow: /book/show/53614255 +Disallow: /book/show/53442209 +Disallow: /book/show/53661728 +Disallow: /book/show/53494663 +Disallow: /book/show/53587446 +Disallow: /book/show/52739939 +Disallow: /book/show/53727512 +Disallow: /book/show/34134943 +Disallow: /book/show/53679620 +Disallow: /book/show/53535688 +Disallow: /book/show/18201005 +Disallow: /book/show/52866283 +Disallow: /book/show/53288782 +Disallow: /book/show/51982892 +Disallow: /book/show/52427527 +Disallow: /book/show/53458164 +Disallow: /book/show/53461075 +Disallow: /book/show/53225490 +Disallow: /book/show/53450795 +Disallow: /book/show/53487064 +Disallow: /book/show/34296466 +Disallow: /book/show/53260660 +Disallow: /book/show/53663056 +Disallow: /book/show/53672658 +Disallow: /book/show/53481224 +Disallow: /book/show/53578973 +Disallow: /book/show/53622615 +Disallow: /book/show/51007940 +Disallow: /book/show/53399690 +Disallow: /book/show/53651842 +Disallow: /book/show/53131641 +Disallow: /book/show/52608787 +Disallow: /book/show/53591236 +Disallow: /book/show/53596052 +Disallow: /book/show/34288751 +Disallow: /book/show/51588320 +Disallow: /book/show/34279455 +Disallow: /book/show/53581128 +Disallow: /book/show/53591268 +Disallow: /book/show/53452488 +Disallow: /book/show/52662402 +Disallow: /book/show/53605600 +Disallow: /book/show/53673289 +Disallow: /book/show/53430616 +Disallow: /book/show/53634236 +Disallow: /book/show/50990318 +Disallow: /book/show/52862257 +Disallow: /book/show/52270432 +Disallow: /book/show/53435887 +Disallow: /book/show/53279963 +Disallow: /book/show/53529515 +Disallow: /book/show/53363939 +Disallow: /book/show/51585037 +Disallow: /book/show/18065655 +Disallow: /book/show/53561359 +Disallow: /book/show/53621061 +Disallow: /book/show/51446241 +Disallow: /book/show/52562709 +Disallow: /book/show/48083685 +Disallow: /book/show/53593596 +Disallow: /book/show/34290201 +Disallow: /book/show/53214622 +Disallow: /book/show/53579245 +Disallow: /book/show/53546617 +Disallow: /book/show/53369445 +Disallow: /book/show/53363950 +Disallow: /book/show/53175188 +Disallow: /book/show/53252711 +Disallow: /book/show/53628367 +Disallow: /book/show/53561351 +Disallow: /book/show/53581132 +Disallow: /book/show/53310958 +Disallow: /book/show/38401626 +Disallow: /book/show/53493462 +Disallow: /book/show/51425819 +Disallow: /book/show/23147565 +Disallow: /book/show/53561626 +Disallow: /book/show/51128228 +Disallow: /book/show/34288752 +Disallow: /book/show/53253782 +Disallow: /book/show/34134944 +Disallow: /book/show/53321001 +Disallow: /book/show/53490836 +Disallow: /book/show/52662427 +Disallow: /book/show/53449215 +Disallow: /book/show/52737472 +Disallow: /book/show/53666485 +Disallow: /book/show/53388914 +Disallow: /book/show/18232800 +Disallow: /book/show/52867698 +Disallow: /book/show/53976087 +Disallow: /book/show/44665940 +Disallow: /book/show/52087144 +Disallow: /book/show/44294481 +Disallow: /book/show/55539602 +Disallow: /book/show/48749728 +Disallow: /book/show/55453281 +Disallow: /book/show/55188757 +Disallow: /book/show/56182927 +Disallow: /book/show/53895780 +Disallow: /book/show/56436827 +Disallow: /book/show/53855549 +Disallow: /book/show/41149445 +Disallow: /book/show/54650709 +Disallow: /book/show/45167506 +Disallow: /book/show/56426014 +Disallow: /book/show/55617338 +Disallow: /book/show/56188292 +Disallow: /book/show/54750926 +Disallow: /book/show/53872927 +Disallow: /book/show/45310608 +Disallow: /book/show/56128876 +Disallow: /book/show/56416598 +Disallow: /book/show/55996851 +Disallow: /book/show/55831668 +Disallow: /book/show/54171648 +Disallow: /book/show/55996842 +Disallow: /book/show/56019526 +Disallow: /book/show/55770816 +Disallow: /book/show/55640907 +Disallow: /book/show/55703297 +Disallow: /book/show/43489513 +Disallow: /book/show/55702973 +Disallow: /book/show/55428019 +Disallow: /book/show/55494000 +Disallow: /book/show/55439818 +Disallow: /book/show/54389597 +Disallow: /book/show/55756732 +Disallow: /book/show/51501236 +Disallow: /book/show/54838300 +Disallow: /book/show/55166076 +Disallow: /book/show/55021284 +Disallow: /book/show/54866372 +Disallow: /book/show/55095022 +Disallow: /book/show/54888492 +Disallow: /book/show/43725570 +Disallow: /book/show/53841183 +Disallow: /book/show/53891971 +Disallow: /book/show/41556469 +Disallow: /book/show/53862761 +Disallow: /book/show/41149455 +Disallow: /book/show/53405333 +Disallow: /book/show/41043686 +Disallow: /book/show/52773236 +Disallow: /book/show/53524193 +Disallow: /book/show/55213963 +Disallow: /book/show/49491850 +Disallow: /book/show/46568698 +Disallow: /book/show/53621325 +Disallow: /book/show/48495767 +Disallow: /book/show/44294721 +Disallow: /book/show/45167507 +Disallow: /book/show/49471529 +Disallow: /book/show/49834025 +Disallow: /book/show/45067687 +Disallow: /book/show/45999616 +Disallow: /book/show/50277983 +Disallow: /book/show/44295375 +Disallow: /book/show/49499737 +Disallow: /book/show/40817434 +Disallow: /book/show/530270 + +Sitemap: https://www.goodreads.com/siteindex.author.xml +Sitemap: https://www.goodreads.com/siteindex.author_community_question.xml +Sitemap: https://www.goodreads.com/siteindex.award.xml +Sitemap: https://www.goodreads.com/siteindex.blog.xml +Sitemap: https://www.goodreads.com/siteindex.book_community_question.xml +Sitemap: https://www.goodreads.com/siteindex.genre.xml +Sitemap: https://www.goodreads.com/siteindex.giveaway.xml +Sitemap: https://www.goodreads.com/siteindex.group.xml +Sitemap: https://www.goodreads.com/siteindex.index.xml +Sitemap: https://www.goodreads.com/siteindex.interview.xml +Sitemap: https://www.goodreads.com/siteindex.list.xml +Sitemap: https://www.goodreads.com/siteindex.quiz.xml +Sitemap: https://www.goodreads.com/siteindex.quote.xml +Sitemap: https://www.goodreads.com/siteindex.quote_tag.xml +Sitemap: https://www.goodreads.com/siteindex.related_work.xml +Sitemap: https://www.goodreads.com/siteindex.topic.xml +Sitemap: https://www.goodreads.com/siteindex.trivia.xml +Sitemap: https://www.goodreads.com/siteindex.user.xml +Sitemap: https://www.goodreads.com/siteindex.video.xml \ No newline at end of file diff --git a/Tests/unit_tests.py b/Tests/unit_tests.py new file mode 100644 index 0000000000000000000000000000000000000000..a8ff0dc39eff0a833358aa7fe5a174b64d20cd92 --- /dev/null +++ b/Tests/unit_tests.py @@ -0,0 +1,141 @@ +""" +Tests +""" +import unittest +import time +from scraper import scrape_book_page, scrape_author_page, url_in_robots_file +from command import check_if_author, check_if_book + +class Test(unittest.TestCase): + """ + Test class for most functionality aside from database handling + """ + def test_url_in_robots_file(self): + """ + test + """ + url = 'https://www.goodreads.com/book/show/52976360-coronavirus-and-christ' + flag = url_in_robots_file(url) + self.assertEqual(flag, True) + def test_url_in_robots_file2(self): + """ + test + """ + url = 'https://www.goodreads.com/book/show/3735293-clean-code' + flag = url_in_robots_file(url) + self.assertEqual(flag, False) + def test_scrape_book_page(self): + """ + test + """ + url = 'https://www.goodreads.com/book/show/3735293-clean-code' + ret_arr = scrape_book_page(url) + self.assertEqual(ret_arr[0], 'https://www.goodreads.com/book/show/3735293-clean-code') + self.assertEqual(ret_arr[1], 'Clean Code: A Handbook of Agile Software Craftsmanship') + self.assertEqual(ret_arr[2], '3735293') + self.assertEqual(ret_arr[3], '9780132350884') + self.assertEqual(ret_arr[4], 'https://www.goodreads.com/author/show/45372.Robert_C_Martin') + self.assertEqual(ret_arr[5], 'Robert C. Martin') + self.assertEqual(ret_arr[9], 'https://i.gr-assets.com/images/S/compressed.photo.goodreads.com/books/1436202607l/3735293._SX318_.jpg') + time.sleep(40) + def test_scrape_book_page2(self): + """ + test + """ + url = 'https://www.goodreads.com/author/show/45372.Robert_C_Martin' + try: + ret_arr = scrape_book_page(url) + except: + ret_arr = None + self.assertEqual(ret_arr, None) + def test_scrape_author_page(self): + """ + test + """ + url = 'https://www.goodreads.com/author/show/45372.Robert_C_Martin' + ret_arr = scrape_author_page(url) + self.assertEqual(ret_arr[0], 'Robert C. Martin') + self.assertEqual(ret_arr[1], 'https://www.goodreads.com/author/show/45372.Robert_C_Martin') + self.assertEqual(ret_arr[2], '45372') + #self.assertEqual(retArr[3], '4.34') + #self.assertEqual(retArr[4], '31522') + #self.assertEqual(retArr[5], '2093') + self.assertEqual(ret_arr[6], 'https://images.gr-assets.com/authors/1490470967p5/45372.jpg') + time.sleep(40) + def test_scrape_author_page2(self): + """ + test + """ + url = 'https://www.goodreads.com/book/show/3735293-clean-code' + try: + ret_arr = scrape_author_page(url) + except: + ret_arr = None + self.assertEqual(ret_arr, None) + def test_handle_check_author(self): + """ + test + """ + author = {"name": 'bob', + "author_url" : 'bob.com', + "author_id" : '1234', + "rating" : '4321', + "rating_count" : '3333', + "review_count" : '4444', + "image_url" : 'bobimg.com', + "related_authors" : '', + "author_books" : ''} + flag = check_if_author(author) + self.assertEqual(flag, True) + def test_handle_check_author2(self): + """ + test + """ + book = {"book_url": 'book.com', + "title" : 'book', + "book_id" : '123', + "ISBN" : '321', + "author_url" : 'bob.com', + "author" : 'bob', + "rating" : '133', + "rating_count" : '412', + "review_count" : '135', + "image_url" : 'bookimg.com', + "similar_books" : ''} + flag = check_if_author(book) + self.assertEqual(flag, False) + def test_handle_check_book(self): + """ + test + """ + book = {"book_url": 'book.com', + "title" : 'book', + "book_id" : '123', + "ISBN" : '321', + "author_url" : 'bob.com', + "author" : 'bob', + "rating" : '133', + "rating_count" : '412', + "review_count" : '135', + "image_url" : 'bookimg.com', + "similar_books" : ''} + flag = check_if_book(book) + self.assertEqual(flag, True) + def test_handle_check_book2(self): + """ + test + """ + author = {"name": 'bob', + "author_url" : 'bob.com', + "author_id" : '1234', + "rating" : '4321', + "rating_count" : '3333', + "review_count" : '4444', + "image_url" : 'bobimg.com', + "related_authors" : '', + "author_books" : ''} + flag = check_if_book(author) + self.assertEqual(flag, False) +if __name__ == "__main__": + unittest.main() + \ No newline at end of file diff --git a/src/command.py b/src/command.py new file mode 100644 index 0000000000000000000000000000000000000000..c2baf9d8361f13c2c81b3635fe07bf94c3e91edb --- /dev/null +++ b/src/command.py @@ -0,0 +1,130 @@ +""" +Implements the command line interface so that the user can choose inputs for the program. +""" +import argparse +import time +import json +import pymongo +from scraper import scrape_author_page, scrape_book_page, url_in_robots_file +from database import database_author_handler, handle_read_authors, handle_read_books, database_book_handler, get_key + + +def check_if_book(document): + """ + Function to see if a Document is a Book + """ + return bool('book_url' in document.keys()) +def check_if_author(document): + """ + Function to see if a Document is an Author + """ + return bool('author_id' in document.keys()) +def read_json(arg): + """ + Function to handle reading a JSON File and updating + or creating new entries in the MongoDB database + """ + data = None + if arg.JSON_File_In is not None: + file = open(arg.JSON_File_In, 'r') + data = json.load(file) + if isinstance(data, dict): + if check_if_author(data): + handle_read_authors(data) + else: + handle_read_books(data) + else: + for entry in data: + if check_if_author(entry): + handle_read_authors(entry) + else: + handle_read_books(entry) + file.close() +def write_json(arg): + """ + Function to handle writing to a JSON File using JSON Data + being exported from the MongoDB database. + """ + if (args.JSON_File_Out is not None and args.input is not None) : + client = pymongo.MongoClient(get_key()) + data_base = client['GoodReadData'] + books = data_base["Books"] + authors = data_base["Authors"] + for book in books.find(): + if arg.input == str(book['book_url']): + with open(arg.JSON_File_Out, "w") as outfile: + json.dump(str(book), outfile, indent = 4) + break + for author in authors.find(): + if arg.input == str(author['author_url']): + with open(arg.JSON_File_Out, "w") as outfile: + json.dump(str(author), outfile, indent = 4) + break +def run_book_handler(url, arg): + """ + This function handles scraping, transferring data to MongoDB's + database for the amount of books that the user specifies. + """ + save_book_urls = [] + for index in range(arg.Number_Of_Books): + save_book_urls.append(url) + print('Begin scraping a page') + ret_arr = scrape_book_page(URL) + database_book_handler(ret_arr) + time.sleep(40) + url = ret_arr[10][index] + for second in range(index): + if url == save_book_urls[second]: + url = ret_arr[10][second+1] + increment = 0 + while True: + if not url_in_robots_file(url): + break + else: + increment+=1 + url = ret_arr[10][increment] + print('Ended this scrape') + return ret_arr[4] +def run_author_handler(url, arg): + """ + This function handles scraping, transferring data to MongoDB's + database for the amount of Authors that the user specifies. + """ + save_author_urls = [] + for index in range(arg.Number_Of_Authors): + save_author_urls.append(url) + print('Begin scraping a page') + ret_arr2 = scrape_author_page(url) + database_author_handler(ret_arr2) + time.sleep(40) + url = ret_arr2[7][index] + for increment in range(index+1): + if url == save_author_urls[increment]: + url = ret_arr2[7][increment+1] + print('Ended this scrape') +if __name__ == "__main__": + parser = argparse.ArgumentParser(description = 'Parse a Book') + parser.add_argument('Book_URL', type=str, help = "GoodReads Book URL") + parser.add_argument('Number_Of_Books', type=int, + help = "enter a number of books to scrape (between 1 and 200)") + parser.add_argument('Number_Of_Authors', type=int, + help = "enter a number of Authors to scrape (between 1 and 50)") + parser.add_argument('-in' ,'--JSON_File_In', required = False, + help = "enter a valid JSON File to read from") + parser.add_argument('-existing' ,'--input', required = False, + help = "enter a valid Book URL or Author URL to Export") + parser.add_argument('-out' ,'--JSON_File_Out', required = False, + help = "enter a JSON File to write to") + args = parser.parse_args() + if url_in_robots_file(args.Book_URL): + print("The given url is contained within the robots.txt file and should not be used") + if 'book' in args.Book_URL: + if args.Number_Of_Books > 200 and args.Number_Of_Authors > 50: + print("Scraping more than 200 Books and 50 Authors can damage the scraped website") + URL = args.Book_URL + authorURL = run_book_handler(URL, args) + run_author_handler(authorURL, args) + read_json(args) + write_json(args) + else: + print('Invalid book URL. Must be a valid book URL from Goodreads.com') diff --git a/src/database.py b/src/database.py new file mode 100644 index 0000000000000000000000000000000000000000..6a2acd8c8b8b0d9855bb8b8ff3c1afaed19a8c54 --- /dev/null +++ b/src/database.py @@ -0,0 +1,68 @@ +""" +Implements the transferring of web scraped data to a MongoDB database +""" +import pymongo +import os +from dotenv import load_dotenv + +def get_key(): + """ + This function gets the unique key to access my MongoDB database collection + """ + load_dotenv("./key.env") + return os.getenv("SECRET_KEY") +def handle_read_books(json_file): + """ + This function handles when a book is being created or an already existing book is being updated in the MongoDB database. + """ + client = pymongo.MongoClient(get_key()) + database = client.GoodReadData + collection = database.Books + collection.update(json_file, json_file, upsert = True) +def handle_read_authors(json_file): + """ + This function handles when an author is being created or an already existing book is being updated in the MongoDB database. + """ + client = pymongo.MongoClient(get_key()) + database = client.GoodReadData + collection = database.Authors + collection.update(json_file, json_file, upsert = True) +def database_book_handler(ret_arr1): + """ + This function handles the web scraped data of a book and transfers it into the MongoDB database. + """ + client = pymongo.MongoClient(get_key()) + database = client.GoodReadData + collection = database.Books + book = {"book_url": ret_arr1[0], + "title" : ret_arr1[1], + "book_id" : ret_arr1[2], + "ISBN" : ret_arr1[3], + "author_url" : ret_arr1[4], + "author" : ret_arr1[5], + "rating" : ret_arr1[6], + "rating_count" : ret_arr1[7], + "review_count" : ret_arr1[8], + "image_url" : ret_arr1[9], + "similar_books" : ret_arr1[10]} + collection.update(book, book, upsert = True) +def database_author_handler(ret_arr1): + """ + This function handles the web scraped data of an Author and transfers it into the MongoDB database. + """ + client = pymongo.MongoClient(get_key()) + database = client.GoodReadData + collection = database.Authors + author= {"name": ret_arr1[0], + "author_url" : ret_arr1[1], + "author_id" : ret_arr1[2], + "rating" : ret_arr1[3], + "rating_count" : ret_arr1[4], + "review_count" : ret_arr1[5], + "image_url" : ret_arr1[6], + "related_authors" : ret_arr1[7], + "author_books" : ret_arr1[8],} + collection.update(author, author, upsert = True) +if __name__ == "__main__": + print(get_key()) + \ No newline at end of file diff --git a/src/scraper.py b/src/scraper.py new file mode 100644 index 0000000000000000000000000000000000000000..36eb1c9d1ed514e5888671236b8a2719ad868b73 --- /dev/null +++ b/src/scraper.py @@ -0,0 +1,81 @@ +""" +Implements the scraping of book and author data from GoodReads.com +""" +import re +import requests +from bs4 import BeautifulSoup as bs + +def url_in_robots_file(url): + """ + This function checks if a given book URL is listed within + the robots.txt file of GoodReads.com. + Returns true if there is a URL given that matches a line + in the robots.txt but returns false otherwise. + """ + file = open('robots.txt', 'r') + while True: + line = file.readline() + if line == "": + file.close() + return False + if '/book/show/' in line: + parse = line.split('/') + if parse[3].strip() in url: + file.close() + return True +def scrape_book_page(url): + """ + Function that handles parsing and scraping a book page from a + generic GoodReads.com Book page. + Returns all the necessary values required for data storage. + """ + book_url = url + request = requests.get(url) + soup = bs(request.text, 'html.parser') + title = soup.find('h1', id='bookTitle').string.strip() + book_id = soup.find('input', id='book_id')['value'] + if soup.find('span', itemprop='isbn').string is None: + isbn = "does not exist" + isbn = soup.find('span', itemprop='isbn').string + author_url = soup.find('a', class_='authorName')['href'] + author = soup.find('span', itemprop='name').string + rating = soup.find('span', itemprop='ratingValue').string.strip() + rating_count = soup.find('meta', itemprop='ratingCount')['content'] + review_count = soup.find('meta', itemprop='reviewCount')['content'] + image_url = soup.find('img', id='coverImage')['src'] + temp = soup.findAll('li', class_='cover') + similar_books = [] + for search in temp: + for child in search.find_all('a'): + similar_books.append(child['href']) + return book_url, title, book_id, isbn, author_url, author, rating, rating_count, review_count, image_url, similar_books +def scrape_author_page(url): + """ + Function that handles parsing and scraping an author page from + a generic GoodReads.com author page. + Returns all the necessary values required for data storage. + """ + request = requests.get(url) + soup = bs(request.text, 'html.parser') + name = soup.find('span', itemprop='name').string + author_url = url + author_id = re.findall("\d+", url)[0] + rating = soup.find('span', itemprop='ratingValue').string + rating_count = soup.find('span', class_="value-title")['title'] + review_count = soup.find('span', class_='count').find('span')['title'] + image_url = soup.find('div', class_='leftContainer authorLeftContainer').find('img')['src'] + tmp = soup.find('div', class_='hreview-aggregate').findAll('a')[1]['href'] + inp = 'http://goodreads.com' + inp += tmp + request2 = requests.get(inp) + soup2 = bs(request2.text, 'html.parser') + related_authors = [] + authors_urls = soup2.findAll('a', class_="gr-h3 gr-h3--serif gr-h3--noMargin") + for authors in authors_urls: + related_authors.append(authors['href']) + temp = soup.findAll('td', width ='5%') + author_books = [] + for search in temp: + for child in search.findAll('a'): + author_books.append(child['href']) + return name, author_url, author_id, rating, rating_count, review_count, image_url, related_authors, author_books