[Python] - 웹 크롤링 (selenium/beautifulsoup4 예제)

 

import os
from urllib import request
from urllib.error import HTTPError
from selenium import webdriver
from selenium.webdriver.common.by import By

url = "https://www.atlassian.com/software/confluence/download-archives"
site = "https://www.atlassian.com"
versionList = set()

# Selenium WebDriver
driver = webdriver.Chrome()
driver.get('https://www.atlassian.com/software/confluence/download-archives')

# 다운로드 method
def get_download(url, fname, directory):
    try:
        # 해당 directory로 전환 (dir 생성되어 있어야 함)
        os.chdir(directory)
        # request 다운로드 링크 
        request.urlretrieve(url,fname)
        print(f'다운로드 완료 : {fname}\n')
    except HTTPError as e:
        print(f'다운로드 에러 : {fname}\n')
        return

def main():
    a_download = driver.find_elements(By.XPATH, "//div[@class='download-one-version']/a")
    for dl in a_download:
        versionList.add(dl.get_attribute("data-version"))

    for v in versionList:
            # https://www.atlassian.com/software/confluence/downloads/binary/atlassian-confluence-7.19.1-x64.exe
            # https://www.atlassian.com/software/confluence/downloads/binary/atlassian-confluence-7.0.5.tar.gz
            downloadURL = "https://www.atlassian.com/software/confluence/downloads/binary/atlassian-confluence-" + v + ".tar.gz"
            downloadFileName = "atlassian-confluence-" + v + ".tar.gz"
            downloadDir = "D:/Atlassian-download/Confluence/" + str(v).split('.')[0]
            get_download(downloadURL, downloadFileName, downloadDir)

if __name__ == "__main__":
    main()

해당 다운로드 페이지가 동적 페이지라 다운로드 링크까지 가져오기 어려웠다.
Selenium 이용하여 동적으로 크롤링하려고 했으나, 실패함. 개선 가능할 것으로 생각된다.
beautifulsoup4는 동적 페이지 크롤링 X, selenium과 함께 사용