
## Web Scraping
## Vladimir Visbek

import urllib.request
from bs4 import BeautifulSoup
import re
from pandas import DataFrame

# all extracted information is stored into these arrays
nimed = []
kohad = []
linnad = []
paevad = []
aastad = []

# In a loop surfes through pages of the website
for i in range(1, 440):
#wiki = "http://kultuur.info/arhiiv/?page=1&todate=2012-12-31&fromdate=2012-01-01&organizer_or_artist_id=&results_on_page=50&query=&location_id=&opportunity=0&locations=country-49"
    wiki = "http://www.culture.ee/archive/?page="+str(i)+"&todate=2014-12-31&fromdate=2010-01-01&organizer_or_artist_id=&query=&local=1&opportunity=0&topic=1&topic=2&topic=4&topic=22&topic=6&topic=7&topic=8&topic=9&topic=149&topic=150&topic=23&topic=24&topic=25&topic=26&topic=283&topic=164&topic=43&topic=44&topic=45&topic=46&topic=47&topic=77&topic=104&topic=122"
    page = urllib.request.urlopen(wiki)
    soup = BeautifulSoup(page, "html.parser")
    all_events = soup.find_all("div", class_="event")

    table = soup.find_all("tr") # as the needed information is on the page in a HTML table, we look for <tr>
    print(table)
    for tr in table:
        col2 = tr.find_all("td", class_="col_2")
        nimed.append(tr.div.div.a.string) # extracting the name of the event
        for c in col2:
            koht = c.a.br.previous_sibling.string
            linn = c.a.br.next_sibling.string
            kohad.append(koht.replace('\n','').replace('\t','')) # location of the event
            linnad.append(linn.replace('\n','').replace('\t','')) # city where the event takes place
        col3 = tr.find_all("td", class_="col_3")
        for c in col3:
            paevad.append(c.p.string)
            aastad.append(c.p.string[6:][:4]) # date of the event

# save-ing into an Excel file
df = DataFrame({'Üritus': nimed, 'Asukoht': kohad, 'Linn': linnad, 'Kuupäev': paevad, 'Aasta': aastad})
df.to_excel('uued_yritused.xlsx', sheet_name='sheet1', index=False)

