412 lines
15 KiB
Python
412 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
import sys, time, codecs, re
|
|
import json
|
|
import yaml
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from bs4 import element
|
|
|
|
host = 'https://en.wikipedia.org'
|
|
|
|
# https://de.wikipedia.org/wiki/Bevölkerungsentwicklung
|
|
# Kennzeichnend für die Bevölkerungsentwicklung der Welt, insbesondere die der letzten 200 Jahre, war ein starkes hyperexponentielles Wachstum, weshalb man auch von Bevölkerungsexplosion spricht. Seit dem Wendepunkt 1962/63 sinkt hingegen die Wachstumsrate und seit 1989 auch der absolute Zuwachs.
|
|
# Bedingt durch seinen negativen Einfluss auf die begrenzte Tragfähigkeit der Erde (siehe auch Ökologischer Fußabdruck) sowie seine Multiplikatorfunktion aller der nachhaltigen Entwicklung entgegenstehenden Aktivitäten des Menschen ist das Bevölkerungswachstum eines der zentralen globalen Probleme und mitverantwortlich für die globale Erwärmung.
|
|
|
|
# https://www.cia.gov/library/publications/the-world-factbook/geos/print_af.html
|
|
|
|
# https://www.webuildinternet.com/2015/07/09/geojson-data-of-the-netherlands/
|
|
|
|
def first_int(td):
|
|
if not td:
|
|
return -1
|
|
raw_str = td.text.replace(',', '')
|
|
int_str = re.search('^\d+', raw_str)
|
|
if int_str:
|
|
return int_str.group(0)
|
|
|
|
return -1
|
|
|
|
def readinfobox(url):
|
|
|
|
pop_total = 0
|
|
area_km2 = 0
|
|
|
|
print('reading infobox from %s ... ' % url)
|
|
soup = BeautifulSoup(html(url), 'html5lib')
|
|
table = soup.find("table", class_="infobox")
|
|
th = table.find_all('th')
|
|
for item in th:
|
|
if item.text.startswith('Area') and area_km2 == 0:
|
|
next_tr = item.parent.next_sibling.next_sibling
|
|
head = next_tr.find('th')
|
|
# print(head)
|
|
# if head:
|
|
# title = re.search('[A-Za-z]+', head.text).group(0).lower()
|
|
td = head.next_sibling.next_sibling
|
|
if td:
|
|
area_km2 = re.search('^[0-9.,]+', td.text.replace(',', '')).group(0)
|
|
elif item.text.startswith('Population'):
|
|
next_tr = item.parent.next_sibling.next_sibling
|
|
pop_total = first_int(next_tr.find('td'))
|
|
|
|
if area_km2 == 0:
|
|
print('error: no area data found')
|
|
|
|
return area_km2, pop_total
|
|
|
|
def readinfobox_raw(url):
|
|
|
|
pop_total = 0
|
|
area_raw = 0
|
|
|
|
# print('reading infobox from %s ... ' % url)
|
|
soup = BeautifulSoup(html(url), 'html5lib')
|
|
table = soup.find("table", class_="infobox")
|
|
th = table.find_all('th')
|
|
for item in th:
|
|
if item.text.startswith('Area') and area_raw == 0:
|
|
|
|
if item.next_sibling.next_sibling != None:
|
|
if item.next_sibling.next_sibling.text != None:
|
|
# print(item.next_sibling.next_sibling)
|
|
nearby_col_text = item.next_sibling.next_sibling.text
|
|
if re.search('km2', nearby_col_text):
|
|
area_raw = nearby_col_text
|
|
|
|
next_row = item.parent.next_sibling.next_sibling
|
|
next_row_head = next_row.find('th')
|
|
next_row_col = next_row_head.next_sibling.next_sibling
|
|
|
|
if next_row_col:
|
|
next_row_col_text = next_row_col.text
|
|
if re.search('km2', next_row_col_text):
|
|
area_raw = next_row_col_text
|
|
|
|
|
|
|
|
|
|
# print(nearby_col_text, next_row_nearby_col_text)
|
|
|
|
# print(head)
|
|
# if head:
|
|
# title = re.search('[A-Za-z]+', head.text).group(0).lower()
|
|
# td = head.next_sibling.next_sibling
|
|
# if td:
|
|
# area_raw = td.text
|
|
# area_raw = re.search('^[0-9.,]+', td.text.replace(',', '')).group(0)
|
|
elif item.text.startswith('Population'):
|
|
if item.next_sibling.next_sibling != None:
|
|
if item.next_sibling.next_sibling.text != None:
|
|
nearby_col_text = item.next_sibling.next_sibling.text
|
|
if re.search('[0-9,]+', nearby_col_text):
|
|
pop_total = re.search('[0-9,]+', nearby_col_text).group(0)
|
|
else:
|
|
if item.parent.next_sibling.next_sibling != None:
|
|
next_row_text = item.parent.next_sibling.next_sibling.find('td').text
|
|
if next_row_text != None:
|
|
if re.search('[0-9,]+', next_row_text):
|
|
pop_total = re.search('[0-9,]+', next_row_text).group(0)
|
|
# else:
|
|
# pop_total = first_int(next_tr.find('td'))
|
|
|
|
if area_raw != 0:
|
|
# filter the number before 'km2'
|
|
area = None
|
|
try:
|
|
area = re.search('[0-9,\.]+?(?=\skm2)', area_raw).group(0)
|
|
except:
|
|
pass
|
|
|
|
if area != None:
|
|
area_raw = area.replace(',', '')
|
|
|
|
if pop_total != 0:
|
|
pop_total = pop_total.replace(',', '')
|
|
|
|
if area_raw == 0:
|
|
print('error: no area data found')
|
|
if pop_total == 0:
|
|
print('error: no population data found')
|
|
if area_raw != 0 and pop_total != 0:
|
|
print('area = <%s>' % area_raw, 'population = <%s>' % pop_total)
|
|
|
|
return area_raw, pop_total
|
|
|
|
def readlistofstates():
|
|
# table xpath sample
|
|
# //*[@id="mw-content-text"]/div/table[1]/tbody/tr[4]/td[1]/b/a
|
|
url = '%s/wiki/List_of_sovereign_states' % host
|
|
request = requests.get(url)
|
|
if not request:
|
|
print('error: could not read contents from <%s>' % url)
|
|
return False
|
|
soup = BeautifulSoup(request.text, 'html5lib')
|
|
table = soup.find("table", class_="wikitable")
|
|
states = []
|
|
for tr in table.find_all('tr'):
|
|
# even_numbers = list(filter(lambda x: x % 2 == 0, fibonacci))
|
|
|
|
firsttd = list(filter(lambda x: x != '\n', tr.children))[0] # str_list = list(filter(None, str_list)) # fastest
|
|
firsttda = firsttd.find('a', href=re.compile('/wiki/'))
|
|
if firsttda:
|
|
states.append({ 'name' : firsttda.text, 'url' : firsttda['href'] })
|
|
# for c in tr.children:
|
|
# print(c)
|
|
# for a in table.find_all('a', href=re.compile('/wiki/')):
|
|
# print(a.text, a.href)
|
|
# for tag in soup.find_all(re.compile("^dt|dd")):
|
|
# if tag.get_text(strip=True) != '':
|
|
# result_list.append(tag)
|
|
# return result_list
|
|
return states
|
|
|
|
def readnavbox(url):
|
|
request = requests.get(url)
|
|
if not request:
|
|
print('error: could not read contents from <%s>' % url)
|
|
return False
|
|
soup = BeautifulSoup(request.text, 'html5lib')
|
|
table = soup.find("table", class_="navbox")
|
|
anchors = table.find_all('a')
|
|
provinces = []
|
|
for anchor in anchors:
|
|
provinces.append({ 'name': anchor.text, 'link' : '%s%s' % (host, anchor['href']) })
|
|
return provinces
|
|
|
|
def html(url):
|
|
print('fetching %s ... ' % url)
|
|
request = requests.get(url)
|
|
if not request:
|
|
print('error: could not read contents from <%s>' % url)
|
|
return False
|
|
return request.text
|
|
|
|
def writejson(filename, aobj):
|
|
with open(filename, 'w') as outfile:
|
|
json.dump(aobj, outfile, sort_keys=True, indent=4, separators=(',', ': '))
|
|
|
|
def germanstates():
|
|
request = requests.get('https://en.wikipedia.org/wiki/List_of_statistical_offices_in_Germany')
|
|
if not request:
|
|
print('error: could not read contents from <%s>' % url)
|
|
return False
|
|
soup = BeautifulSoup(request.text, 'html5lib')
|
|
table = soup.find("table", class_="wikitable")
|
|
rows = table.find_all('tr')
|
|
states = []
|
|
for row in rows:
|
|
children = list(row.children)
|
|
if len(children) > 1:
|
|
anchor = children[1].find('a')
|
|
if anchor:
|
|
href = anchor['href']
|
|
name = anchor.text
|
|
states.append({'name_en' : name, 'link' : 'https://en.wikipedia.org%s' % href })
|
|
return states
|
|
|
|
def netherlands():
|
|
# provinces: table class wikitable no 2 + 3, links in left column
|
|
url = 'https://en.wikipedia.org/wiki/Netherlands'
|
|
soup = BeautifulSoup(html(url), 'html5lib')
|
|
wikitables = soup.find_all("table", class_="wikitable")
|
|
tables = [wikitables[1]]
|
|
states = []
|
|
for table in tables:
|
|
tbody = table.find('tbody')
|
|
for row in tbody.find_all('tr'):
|
|
children = list(row.children)
|
|
if len(children) > 3:
|
|
anchor = children[1].find('a')
|
|
if anchor:
|
|
|
|
# normalize names
|
|
name = anchor['title']
|
|
name = re.sub('\(.*\)', '', name) # remove brackets
|
|
name = name.replace('_', '-') # only dashes
|
|
name = name.replace(' ', '-') # only dashes
|
|
if name[-1:] == '_': # remove trailing underscore
|
|
name = name[:-1]
|
|
if name[-1:] == '-': # remove trailing underscore
|
|
name = name[:-1]
|
|
name_nl = name.replace('South', 'Zuid') # in dutch
|
|
name_nl = name_nl.replace('North', 'Noord') # in dutch
|
|
# print(name)
|
|
states.append({'name_nl' : name_nl, 'name_en' : name, 'link' : 'https://en.wikipedia.org%s' % anchor['href'] })
|
|
return states
|
|
|
|
def netherlands_municipalities():
|
|
# provinces: table class wikitable no 2 + 3, links in left column
|
|
url = 'https://en.wikipedia.org/wiki/List_of_municipalities_of_the_Netherlands'
|
|
soup = BeautifulSoup(html(url), 'html5lib')
|
|
wikitables = soup.find_all("table", class_="wikitable")
|
|
# print(len(wikitables))
|
|
# return
|
|
tables = [wikitables[0]]
|
|
states = []
|
|
for table in tables:
|
|
tbody = table.find('tbody')
|
|
for row in tbody.find_all('tr'):
|
|
children = list(row.children)
|
|
if len(children) > 3:
|
|
anchor = children[1].find('a')
|
|
if anchor:
|
|
states.append({'name_en' : anchor['title'], 'link' : 'https://en.wikipedia.org%s' % anchor['href'] })
|
|
return states
|
|
|
|
|
|
def area_population_infoboxes(items):
|
|
for item in items:
|
|
area_km2, pop_total = readinfobox(item['link'])
|
|
item['area_km2'] = area_km2
|
|
item['population'] = pop_total
|
|
return items
|
|
|
|
def listregions(url, tablenum, colnum):
|
|
soup = BeautifulSoup(html(url), 'html5lib')
|
|
wikitables = soup.find_all("table", class_="wikitable")
|
|
print('found %d wikitables'% len(wikitables))
|
|
if len(wikitables) < tablenum:
|
|
print('error: table at index <%d> not available', tablenum)
|
|
return
|
|
tables = [wikitables[tablenum]]
|
|
items = []
|
|
for table in tables:
|
|
tbody = table.find('tbody')
|
|
for row in tbody.find_all('tr'):
|
|
children = list(row.children)
|
|
if len(children) > 3:
|
|
anchor = children[colnum].find('a')
|
|
if anchor != None:
|
|
if anchor.get('title', None) != None and anchor.get('href', None) != None:
|
|
item = {'name' : anchor.text, 'link' : 'https://en.wikipedia.org%s' % anchor['href'] }
|
|
print(item)
|
|
items.append(item)
|
|
return items
|
|
|
|
def main():
|
|
|
|
# readinfobox_raw('https://en.wikipedia.org/wiki/Greater_London')
|
|
# readinfobox_raw('https://en.wikipedia.org/wiki/South_Holland')
|
|
# readinfobox_raw('https://en.wikipedia.org/wiki/Saxony-Anhalt')
|
|
# readinfobox_raw('https://en.wikipedia.org/wiki/Berlin')
|
|
# readinfobox_raw('https://en.wikipedia.org/wiki/Hong_Kong')
|
|
# readinfobox_raw('https://en.wikipedia.org/wiki/Geldermalsen')
|
|
# return
|
|
|
|
items = listregions('https://en.wikipedia.org/wiki/Regions_of_England', 0, 1)
|
|
for item in items:
|
|
area_km2, pop_total = readinfobox_raw(item['link'])
|
|
item['area_km2'] = area_km2
|
|
item['population'] = pop_total
|
|
writejson('eng/regions.json', items)
|
|
return
|
|
# print(readinfobox('https://en.wikipedia.org/wiki/Geldermalsen'))
|
|
# return
|
|
|
|
# writejson('netherlands-municipalities.json', area_population_infoboxes(netherlands_municipalities()))
|
|
# # print(items)
|
|
# return
|
|
|
|
# writejson('netherlands.json', netherlands())
|
|
# return
|
|
|
|
# print(readinfobox('https://en.wikipedia.org/wiki/South_Holland'))
|
|
# print(readinfobox('https://en.wikipedia.org/wiki/Saxony-Anhalt'))
|
|
# print(readinfobox('https://en.wikipedia.org/wiki/Berlin'))
|
|
# print(readinfobox('https://en.wikipedia.org/wiki/Hong_Kong'))
|
|
# return
|
|
|
|
provinces = netherlands()
|
|
for province in provinces:
|
|
area_km2_int, pop_total_int = readinfobox(province['link'])
|
|
province['area_km2'] = area_km2_int
|
|
province['population'] = pop_total_int
|
|
writejson('netherlands/provinces.json', provinces)
|
|
return
|
|
|
|
# area_km2_int, pop_total_int = readinfobox('https://en.wikipedia.org/wiki/Hong_Kong')
|
|
# print(area_km2_int, pop_total_int)
|
|
# return
|
|
|
|
host = 'https://en.wikipedia.org'
|
|
url = '%s/wiki/China' % host
|
|
provinces = readnavbox(url)
|
|
for province in provinces:
|
|
name = province['name']
|
|
link = province['link']
|
|
area_km2_int, pop_total_int = readinfobox(link)
|
|
province['area_km2'] = area_km2_int
|
|
province['population'] = pop_total_int
|
|
|
|
writejson('china.json', provinces)
|
|
return
|
|
|
|
return
|
|
readstatepop()
|
|
return
|
|
states = readlistofstates()
|
|
writejson('en-wikipedia-%s-states.json' % len(states), states)
|
|
return 0
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|
|
|
|
|
|
|
|
|
|
|
|
#### trash
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# normalize names
|
|
# items = readjson('netherlands/netherlands.json')
|
|
# for item in items:
|
|
# name = item['name_en']
|
|
# name = re.sub('\(.*\)', '', name) # remove brackets
|
|
# name = name.replace('_', '-') # only dashes
|
|
# name = name.replace(' ', '-') # only dashes
|
|
# if name[-1:] == '_': # remove trailing underscore
|
|
# name = name[:-1]
|
|
# if name[-1:] == '-': # remove trailing underscore
|
|
# name = name[:-1]
|
|
# name = name.replace('South', 'Zuid') # in english
|
|
# name = name.replace('North', 'Noord') # in english
|
|
# print(name)
|
|
|
|
|
|
# next_tr = item.parent.next_sibling.next_sibling
|
|
# next_tr_td = next_tr.find('td')
|
|
# pop_total_raw_str = next_tr_td.text.replace(',', '')
|
|
# pop_total = re.search('^\d+', pop_total_raw_str)
|
|
# if pop_total:
|
|
# pop_total_int = pop_total.group(0)
|
|
|
|
# next_tr_th = next_tr.find('th')
|
|
# if re.search('Total', next_tr_th.text) or re.search('Municipality', next_tr_th.text):
|
|
# next_tr_td = next_tr_th.next_sibling.next_sibling
|
|
# pop_total_raw_str = next_tr_td.text.replace(',', '')
|
|
# pop_total = re.search('^\d+', pop_total_raw_str)
|
|
# if pop_total:
|
|
# pop_total_int = pop_total.group(0)
|
|
|
|
|
|
# popa = table.find("a", text=re.compile('Population'))
|
|
# area_a = table.find("a", text=re.compile('Area'))
|
|
# # print(popa, area_a)
|
|
# return
|
|
# pop = list(popa.parent.parent.next_sibling.next_sibling.find('td').children)[0]
|
|
# area_km2 = list(area_a.parent.parent.next_sibling.next_sibling.find('td').children)[0]
|
|
# print(pop, area_km2)
|
|
|
|
# pop_int = int(pop.replace(',', ''))
|
|
# area_km2_int = int(area_km2.replace(',', ''))
|
|
# print(pop_int, area_km2_int)
|
|
|
|
|
|
# density = pop_int / area_km2_int
|
|
# print(density) |