Pastebin launched a little side project called HostCabi.net, check it out ;-)Don't like ads? PRO users don't see any ads ;-)
Public Pastes
Guest

FimFiction_downloader 2013-7-25

By: waterapple on Jul 25th, 2013  |  syntax: None  |  size: 9.31 KB  |  hits: 3  |  expires: Never
download  |  raw  |  embed  |  report abuse  |  print
Text below is selected. Please press Ctrl+C to copy to your clipboard. (⌘+C on Mac)
  1. #-------------------------------------------------------------------------------
  2. # Name:        FimFiction_downloader
  3. # Purpose:      Automated bulk download of stories from fimfiction.com
  4. #
  5. # Author:      new
  6. #
  7. # Created:     10/07/2013
  8. # Copyright:   (c) new 2013
  9. # Licence:     <your licence>
  10. #-------------------------------------------------------------------------------
  11. #!/usr/bin/env python
  12.  
  13.  
  14. import mechanize
  15. import cookielib
  16. import re
  17. import json
  18. import imghdr
  19. import os
  20. import time
  21. import string
  22. from urllib2 import urlopen, URLError, HTTPError
  23. from httplib import BadStatusLine
  24. import glob
  25. import random
  26. import logging
  27. from bs4 import BeautifulSoup
  28.  
  29. # getwithinfo()
  30. GET_REQUEST_DELAY = 0
  31. GET_REQUEST_DELAY_UPPER_RANDOM = 1
  32. GET_RETRY_DELAY = 30
  33. GET_MAX_ATTEMPTS = 20
  34.  
  35. LOGGING_LOG_PATH = "FimFiction_Downloader_log.txt"
  36.  
  37. def delay(basetime,upperrandom=10):
  38.     #replacement for using time.sleep, this adds a random delay to be sneaky
  39.     sleeptime = basetime + random.randint(0,upperrandom)
  40.     logger.debug("pausing for "+str(sleeptime)+" ...")
  41.     time.sleep(sleeptime)
  42.  
  43.  
  44. def get(url):
  45.     #try to retreive a url. If unable to return None object
  46.     #Example useage:
  47.     #html = get('')
  48.     #if html:
  49.     gettuple = getwithinfo(url)
  50.     if gettuple:
  51.         reply, info = gettuple
  52.         return reply
  53.  
  54. def getwithinfo(url):
  55.     """Try to retreive a url. If unable to return None objects
  56.     Example useage:
  57.     html = get('')
  58.         if html:
  59.     """
  60.     attemptcount = 0
  61.     while attemptcount < GET_MAX_ATTEMPTS:
  62.         attemptcount = attemptcount + 1
  63.         if attemptcount > 1:
  64.             print 'Attempt', attemptcount
  65.         try:
  66.             r = br.open(url)
  67.             info = r.info()
  68.             reply = r.read()
  69.             delay(GET_REQUEST_DELAY, GET_REQUEST_DELAY_UPPER_RANDOM)
  70.             # Save html responses for debugging
  71.             #print info
  72.             #print info["content-type"]
  73.             if "html" in info["content-type"]:
  74.                 #print "saving debug html"
  75.                 savefile("debug\\get_last_html.htm", reply, True)
  76.             return reply,info
  77.         except HTTPError, err:
  78.             logger.debug(str(err))
  79.             if err.code == 404:
  80.                 logger.debug('404 error')
  81.                 return
  82.             elif err.code == 410:
  83.                 logger.debug('410 error, GONE')
  84.                 return
  85.             else:
  86.                 savefile("\debug\\error.htm",err.fp.read(),True)
  87.         except URLError, err:
  88.             logger.debug(str(err))
  89.             if 'unknown url type:' in err.reason:
  90.                 return
  91.         except BadStatusLine, err:
  92.             logger.debug(str(err))
  93.         delay(GET_RETRY_DELAY)
  94.  
  95. def savefile(filenamein,data,force_save=False):
  96.     if not force_save:
  97.         if os.path.exists(filenamein):
  98.             logger.debug("file already exists! "+str(filenamein))
  99.             return
  100.     sanitizedpath = sanitizepath(filenamein)
  101.     foldername = os.path.dirname(sanitizedpath)
  102.     if not os.path.isdir(foldername):
  103.         os.makedirs(foldername)
  104.     file = open(sanitizedpath, "wb")
  105.     file.write(data)
  106.     file.close()
  107.  
  108. def sanitizepath(pathin):
  109.     #from pathsanitizer
  110.     #sanitize a filepath for use on windows
  111.     #http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx
  112.     assert(type(pathin)==type(""))
  113.     segments = []
  114.     workingpath = pathin# make a copy for easier debugging
  115.     #split the path into segments
  116.     while True:
  117.         workingpath, segment = os.path.split(workingpath)
  118.         segments.append(segment)
  119.         if len(workingpath) <= 0:
  120.             break
  121.     segments.reverse()
  122.     #sanitize segments
  123.     precessedsegments = []
  124.     for segment in segments:
  125.         s0 = re.sub('[^A-Za-z0-9\ \.\_]+', '-', segment)#remove all non-alphanumeric
  126.         s1 = s0.strip()#strip whitespace so it doesn't get turned into hyphens
  127.         s2 = re.sub('[<>:"/\|?*]+', '-',s1)#remove forbidden characters
  128.         s3 = s2.strip()#strip whitespace
  129.         s4 = s3.strip(".-")#strip characters that shouldn't be at ends of filenames
  130.         s5 = re.sub(r"\ +", " ", s4)#remove repeated spaces
  131.         s6 = re.sub(r"\-+", "-", s5)#remove repeated hyphens
  132.         s7 = re.sub(r"\_+", "_", s6)#remove repeated underscores
  133.         precessedsegments.append(s7)
  134.     #join segments
  135.     pathout = os.path.join(*precessedsegments)
  136.     assert(type(pathout)==type(""))
  137.     return pathout
  138.  
  139. def setupbrowser():
  140.     #Initialize browser object to global variable "br" using cokie jar "cj"
  141.     # Browser
  142.     global br
  143.     br = mechanize.Browser()
  144.     br.set_cookiejar(cj)
  145.     # Browser options
  146.     br.set_handle_equiv(True)
  147.     br.set_handle_gzip(True)
  148.     br.set_handle_redirect(True)
  149.     br.set_handle_referer(True)
  150.     br.set_handle_robots(False)
  151.     # Follows refresh 0 but not hangs on refresh > 0
  152.     br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
  153.     # User-Agent (this is cheating, ok?)
  154.     br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
  155.  
  156.  
  157.  
  158. def extract_filename_from_http_metadata(metadata):
  159.     # Read filename from metadata
  160.     #print metadata
  161.     filename = metadata['content-disposition'].split('filename=')[1][1:][:-1]
  162.     filename = re.sub(r"[/\\]", r"", filename)# remove slashes to stop confusion
  163.  
  164.     # crop filename to 100 chars plus extention if length of filename is over 100
  165.     if len(filename) > 100:
  166.         filename_no_extention, file_extention = os.path.splitext(filename)
  167.         length_to_crop_to = 100
  168.         marker_string = "---"
  169.         allowed_number_of_filename_chars = (length_to_crop_to - len(file_extention) - len(marker_string))
  170.         cropped_filename_no_extention = filename_no_extention[0:allowed_number_of_filename_chars]
  171.         cropped_filename = cropped_filename_no_extention + marker_string + file_extention
  172.         filename = cropped_filename
  173.     return filename
  174.  
  175. def make_story_filename(story_id_string,download_dir_path,download_metadata):
  176.     remote_filename = extract_filename_from_http_metadata(download_metadata)
  177.     save_filename = story_id_string+" - "+remote_filename
  178.     save_path = os.path.join(download_dir_path, save_filename)
  179.     logger.debug(save_path)
  180.     return save_path
  181.  
  182. def download_story(download_dir_path, story_id):
  183.     """Download a story using fimfics download urls"""
  184.     # Change storyid integer into a string stuitable for download links
  185.     story_id_string = str(story_id).rjust(6, "0")
  186.     # Form download urls
  187.     story_html_download_url = "https://www.fimfiction.net/download_story.php?story="+story_id_string+"&html"
  188.     story_txt_download_url = "https://www.fimfiction.net/download_story.php?story="+story_id_string
  189.     story_epub_download_url = "https://www.fimfiction.net/download_epub.php?story="+story_id_string
  190.     download_urls = [story_html_download_url, story_txt_download_url, story_epub_download_url]
  191.  
  192.     # Download story urls
  193.     for download_url in download_urls:
  194.         # Load url
  195.         download_data, download_info = getwithinfo(download_url)
  196.         if not download_data:
  197.             return False
  198.         save_path = make_story_filename(story_id_string, download_dir_path, download_info)
  199.         savefile(save_path, download_data)
  200.     return True
  201.  
  202.  
  203. def download_stories(download_dir_path,start_number,stop_number):
  204.     logger.debug("downloading stories from "+str(start_number)+" to "+str(stop_number))
  205.     story_counter = start_number
  206.     max_total_skipped = 10000
  207.     max_sequential_skipped = 100
  208.     total_skipped_counter = 0
  209.     sequential_skipped = 0
  210.     while (story_counter <= stop_number) and (total_skipped_counter <= max_total_skipped) and (sequential_skipped <=  max_sequential_skipped):
  211.         logger.debug("downloading story id: "+str(story_counter))
  212.         download_success = download_story(download_dir_path,story_counter)
  213.         if not download_success:
  214.             total_skipped_counter += 1
  215.             sequential_skipped += 1
  216.             logger.debug("download failed, sequential fails: "+str(sequential_skipped)+" total fails:"+str(total_skipped_counter))
  217.         else:
  218.             sequential_skipped = 0
  219.             logger.debug("download succeeded.")
  220.  
  221.         story_counter += 1
  222.  
  223. def main():
  224.     date_of_run_start = time.strftime('%Y %m')
  225.     download_folder = os.path.join("download", date_of_run_start)
  226.     download_stories(download_folder, 25110, 2000000)
  227.  
  228.  
  229. # Setup logging (Before running any other code)
  230. # http://inventwithpython.com/blog/2012/04/06/stop-using-print-for-debugging-a-5-minute-quickstart-guide-to-pythons-logging-module/
  231. logger = logging.getLogger()
  232. logger.setLevel(logging.DEBUG)
  233. formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
  234. fh = logging.FileHandler(LOGGING_LOG_PATH)
  235. fh.setLevel(logging.DEBUG)
  236. fh.setFormatter(formatter)
  237. logger.addHandler(fh)
  238. ch = logging.StreamHandler()
  239. ch.setLevel(logging.DEBUG)
  240. ch.setFormatter(formatter)
  241. logger.addHandler(ch)
  242. logger.debug('Logging started.')
  243.  
  244. if __name__ == '__main__':
  245.     try:
  246.         cj = cookielib.LWPCookieJar()
  247.         setupbrowser()
  248.         main()
  249.         logger.debug("End of run.")
  250.      # Log exceptions
  251.     except Exception, e:
  252.         logger.critical("Unhandled exception!")
  253.         logging.exception(e)
create a new version of this paste RAW Paste Data