104 lines
3.8 KiB
Python
104 lines
3.8 KiB
Python
#!/usr/bin/env python3
|
|
import sys, time, re, io, csv
|
|
import requests
|
|
from zipfile import ZipFile, is_zipfile
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
|
with open("data/world/countries.iso3.json") as file:
|
|
countries = json.loads(file.read())
|
|
for key, value in countries.items():
|
|
|
|
country_iso_code = value
|
|
csvpath = 'data/%s/databank.worldbank.org' % country_iso_code
|
|
|
|
url1 = 'http://databank.worldbank.org/data/Views/AjaxDownload/FileInfoHandler.ashx'
|
|
headers1 = {
|
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/604.3.5 (KHTML, like Gecko) Version/11.0.1 Safari/604.3.5',
|
|
'Content-Type' : 'application/x-www-form-urlencoded',
|
|
'X-Requested-With' : 'XMLHttpRequest',
|
|
'Accept-Encoding' : 'gzip, deflate',
|
|
'Accept' : '*/*'
|
|
}
|
|
payload1 = 'reqtype=csv&from=widget-preview&reportid=1516&reporttype=Table&internal=y&country=%s&lang=en' % country_iso_code
|
|
|
|
with requests.Session() as s:
|
|
|
|
r = s.post(url1, headers=headers1, data=payload1, timeout=3)
|
|
|
|
print(country_iso_code, r.status_code, r.encoding, r.text)
|
|
|
|
if r.status_code == 200:
|
|
|
|
filename = r.text
|
|
url2 = 'http://databank.worldbank.org/data/Views/AjaxDownload/FileDownloadHandler.ashx?%s' % filename
|
|
headers2 = {
|
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/604.3.5 (KHTML, like Gecko) Version/11.0.1 Safari/604.3.5',
|
|
'Accept-Encoding' : 'gzip, deflate',
|
|
'Connection' : 'keep-alive',
|
|
'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
'Accept-Language' : 'en-us',
|
|
'Referer' : 'http://databank.worldbank.org/data/Views/Reports/ReportWidgetCustom.aspx?Report_Name=CountryProfile&Id=b450fd57&tbar=y&dd=y&inf=n&zm=n&country=%s' % country_iso_code
|
|
}
|
|
|
|
r = s.get(url2, headers=headers2, timeout=3)
|
|
|
|
print(country_iso_code, r.status_code, len(r.content))
|
|
|
|
if r.status_code == 200 and r.encoding == None:
|
|
with ZipFile(io.BytesIO(r.content)) as z:
|
|
z.extract(z.namelist()[0], csvpath)
|
|
|
|
time.sleep(2)
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# csvpath = 'data/%s/worldbank.csv' % country_iso_code.lower()
|
|
# csvcontents = ''
|
|
# csvfile = None
|
|
# # b = open('bfa.zip', 'rab')
|
|
# # if is_zipfile(b):
|
|
|
|
# with ZipFile('bfa.zip') as z:
|
|
# csvfile = z.open(z.namelist()[0])
|
|
# # with z.open(z.namelist()[0]) as f:
|
|
# # csvcontents = f.read()
|
|
|
|
# # print(csvcontents)
|
|
# # return
|
|
# with ZipFile('bfa.zip', 'a') as z:
|
|
# # files = z.namelist()
|
|
|
|
# # firstfile = z.open(files[0], 'rU')
|
|
# # print(firstfile)
|
|
# z.writestr(csvfile, 'worldbank.csv')
|
|
# # print(z.namelist())
|
|
# # z.extract(files[0], csvpath)
|
|
# # csvfile = z.open(files[0], newline='').read().decode('utf-8')
|
|
# # for row in csv.reader(csvfile, delimiter=','):
|
|
# # print(row)
|
|
|
|
# with ZipFile('bfa.zip') as z:
|
|
# print(z.namelist())
|
|
# with z.open(z.namelist()[0]) as f:
|
|
# csvcontents = f.read()
|
|
# return
|
|
# with open("data/world/countries.iso3.json") as file:
|
|
# countries = json.loads(file.read())
|
|
# for key, value in countries.items():
|
|
# print(key, value)
|
|
|
|
# url = 'https://api.github.com/some/endpoint'
|
|
# headers = {'user-agent': 'my-app/0.0.1'}
|
|
# r = requests.get(url, headers=headers)
|
|
|