How to “webscrape” a site containing a popup window, using python?

Question

I am trying to web scrape a certain part of the etherscan site with python, since there is no api for this functionality. Basically going to this link and one would need to

Answer 1

import requests
from bs4 import BeautifulSoup


def Main(url):
    with requests.Session() as req:
        r = req.get(url, headers={'User-Agent': 'Ahmed American :)'})
        soup = BeautifulSoup(r.content, 'html.parser')
        vs = soup.find("input", id="__VIEWSTATE").get("value")
        vsg = soup.find("input", id="__VIEWSTATEGENERATOR").get("value")
        ev = soup.find("input", id="__EVENTVALIDATION").get("value")
        data = {
            '__VIEWSTATE': vs,
            '__VIEWSTATEGENERATOR': vsg,
            '__EVENTVALIDATION': ev,
            'ctl00$ContentPlaceHolder1$txtContractAddress': '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48',
            'ctl00$ContentPlaceHolder1$btnSubmit': "Verify"
        }
        r = req.post(
            "https://etherscan.io/proxyContractChecker?a=0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", data=data, headers={'User-Agent': 'Ahmed American :)'})
        soup = BeautifulSoup(r.content, 'html.parser')
        token = soup.find(
            "div", class_="alert alert-success").text.split(" ")[-1]
        print(token)


Main("https://etherscan.io/proxyContractChecker")

Output:

0x0882477e7895bdc5cea7cb1552ed914ab157fe56