Sometimes you need to scrape info off of a web page that sits behind a cookie login. It can be done with a combo of browsercookie3 and BeautifulSoup. Full example of grabbing links found on a Yahoo page:
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import browser_cookie3
from bs4 import BeautifulSoup
import sys
def main():
dcookie = browser_cookie3.chrome(domain_name='yahoo.com')
myURL='https://yahoo.com'
try:
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
r = requests.get(myURL, cookies=dcookie, verify=False)
soup = BeautifulSoup(r.content, 'html.parser')
for link in soup.find_all('a'):
print(link.get('href'))
except:
print(sys.exc_info()[0])
sys.exit()
if __name__ == '__main__':
main()
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import browser_cookie3
from bs4 import BeautifulSoup
import sys
def main():
dcookie = browser_cookie3.chrome(domain_name='yahoo.com')
myURL='https://yahoo.com'
try:
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
r = requests.get(myURL, cookies=dcookie, verify=False)
soup = BeautifulSoup(r.content, 'html.parser')
for link in soup.find_all('a'):
print(link.get('href'))
except:
print(sys.exc_info()[0])
sys.exit()
if __name__ == '__main__':
main()