Pastebin launched a little side project called HostCabi.net, check it out ;-)Don't like ads? PRO users don't see any ads ;-)
Public Pastes
Guest

derpi dl funcs

By: waterapple on Mar 13th, 2014  |  syntax: Python  |  size: 5.47 KB  |  hits: 76  |  expires: Never
download  |  raw  |  embed  |  report abuse  |  print
Text below is selected. Please press Ctrl+C to copy to your clipboard. (⌘+C on Mac)
  1. def copy_over_if_duplicate(settings,submission_id,output_folder):
  2.     """Check if there is already a copy of the submission downloaded in the download path.
  3.    If there is, copy the existing version to the suppplied output location then return True
  4.    If no copy can be found, return False"""
  5.     assert_is_string(submission_id)
  6.     # Generate expected filename pattern
  7.     expected_submission_filename = "*"+submission_id+".*"
  8.     # Generate search pattern
  9.     glob_string = os.path.join(settings.output_folder, "*", expected_submission_filename)
  10.     # Use glob to check for existing files matching the expected pattern
  11.     glob_matches = glob.glob(glob_string)
  12.     # Check if any matches, if no matches then return False
  13.     if len(glob_matches) == 0:
  14.         return False
  15.     else:
  16.         # If there is an existing version:
  17.         for glob_match in glob_matches:
  18.             # If there is an existing version in the output path, nothing needs to be copied
  19.             if output_folder in glob_match:
  20.                 return False
  21.             else:
  22.                 # Copy over submission file and metadata JSON
  23.                 logging.info("Trying to copy from previous download: "+glob_match)
  24.                 # Check output folders exist
  25.                 # Build expected paths
  26.                 match_dir, match_filename = os.path.split(glob_match)
  27.                 expected_json_input_filename = submission_id+".json"
  28.                 expected_json_input_folder = os.path.join(match_dir, "json")
  29.                 expected_json_input_location = os.path.join(expected_json_input_folder, expected_json_input_filename)
  30.                 json_output_folder = os.path.join(output_folder, "json")
  31.                 json_output_filename = submission_id+".json"
  32.                 json_output_path = os.path.join(json_output_folder, json_output_filename)
  33.                 submission_output_path = os.path.join(output_folder,match_filename)
  34.                 # Redownload if a file is missing
  35.                 if not os.path.exists(glob_match):
  36.                     logging.debug("Submission file to copy is missing.")
  37.                     return False
  38.                 if not os.path.exists(expected_json_input_location):
  39.                     logging.debug("JSON file to copy is missing.")
  40.                     return False
  41.                 # Ensure output path exists
  42.                 if not os.path.exists(json_output_folder):
  43.                     os.makedirs(json_output_folder)
  44.                 if not os.path.exists(output_folder):
  45.                     os.makedirs(output_folder)
  46.                 logging.info("Copying files for submission: "+submission_id+" from "+match_dir+" to "+output_folder)
  47.                 # Copy over files
  48.                 try:
  49.                     # Copy submission file
  50.                     shutil.copy2(glob_match, submission_output_path)
  51.                     # Copy JSON
  52.                     shutil.copy2(expected_json_input_location, json_output_path)
  53.                     return True
  54.                 except IOError, err:
  55.                     logging.error("Error copying files!")
  56.                     logging.exception(err)
  57.                     return False
  58.  
  59.  
  60. def download_submission(settings,search_tag,submission_id):
  61.     """Download a submission from Derpibooru"""
  62.     assert_is_string(search_tag)
  63.     assert_is_string(submission_id)
  64.     setup_browser()
  65.     #logging.debug("Downloading submission:"+submission_id)
  66.     # Build JSON paths
  67.     json_output_filename = submission_id+".json"
  68.     json_output_path = os.path.join(settings.output_folder,search_tag,"json",json_output_filename)
  69.     # Check if download can be skipped
  70.     # Check if JSON exists
  71.     if os.path.exists(json_output_path):
  72.         logging.debug("JSON for this submission already exists, skipping.")
  73.         return
  74.     # Check for dupliactes in download folder
  75.     output_folder = os.path.join(settings.output_folder,search_tag)
  76.     if copy_over_if_duplicate(settings, submission_id, output_folder):
  77.         return
  78.     # Build JSON URL
  79.     json_url = "https://derpibooru.org/"+submission_id+".json?key="+settings.api_key
  80.     # Load JSON URL
  81.     json_page = get(json_url)
  82.     if not json_page:
  83.         return
  84.     # Convert JSON to dict
  85.     json_dict = decode_json(json_page)
  86.     # Check if submission is deleted
  87.     if check_if_deleted_submission(json_dict):
  88.         logging.debug(json_page)
  89.         return
  90.     # Extract needed info from JSON
  91.     image_url = json_dict["image"]
  92.     image_filename = json_dict["file_name"]
  93.     image_file_ext = json_dict["original_format"]
  94.     # Build image output filenames
  95.     if settings.output_long_filenames:
  96.         image_output_filename = settings.filename_prefix+image_filename+"."+image_file_ext
  97.     else:
  98.         image_output_filename = settings.filename_prefix+submission_id+"."+image_file_ext
  99.     image_output_path = os.path.join(output_folder,image_output_filename)
  100.     # Load image data
  101.     authenticated_image_url = image_url+"?key="+settings.api_key
  102.     logging.debug("Loading submission image: "+authenticated_image_url)
  103.     image_data = get(authenticated_image_url)
  104.     if not image_data:
  105.         return
  106.     # Image should always be bigger than this, if it isn't we got a bad file
  107.     if len(image_data) < 100:
  108.         logging.error("Image data was too small! "+str(image_data))
  109.         return
  110.     # Save image
  111.     save_file(image_output_path, image_data, True)
  112.     # Save JSON
  113.     save_file(json_output_path, json_page, True)
  114.     logging.debug("Download successful")
create a new version of this paste RAW Paste Data