454 lines
16 KiB
Python
454 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
import sys, time, codecs, re
|
|
import json
|
|
import yaml
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from bs4 import element
|
|
|
|
|
|
def writejson(filename, aobj):
|
|
with open(filename, 'w') as outfile:
|
|
json.dump(aobj, outfile, sort_keys=True, indent=4, separators=(', ', ': '))
|
|
|
|
def writejsoncompact(filename, aobj):
|
|
with open(filename, 'w') as outfile:
|
|
json.dump(aobj, outfile, sort_keys=True, separators=(',', ':'))
|
|
|
|
def readjson(file):
|
|
print('reading json from %s ...' % file)
|
|
data = json.loads(open(file).read())
|
|
if not data:
|
|
print('error loading data file')
|
|
return False
|
|
return data
|
|
|
|
|
|
def area_fb2000():
|
|
html = open('data/factbook/2000/area.html').read()
|
|
if not html:
|
|
print('error reading area.html')
|
|
return False
|
|
states = []
|
|
soup = BeautifulSoup(html, 'html5lib')
|
|
for p in soup.find_all('p'):
|
|
children = list(p.children)
|
|
# print(children)
|
|
if len(children) >= 11:
|
|
|
|
state = children[0].text[:-1]
|
|
total = str(children[4]).strip().replace(',', '')
|
|
if 'million' in total:
|
|
total_num = float(total.split(' ')[0]) * 1000000
|
|
total = total_num
|
|
else:
|
|
total_num = re.search('\d+', total)
|
|
if total_num:
|
|
total = total_num.group(0)
|
|
|
|
land = str(children[7]).strip().replace(',', '')
|
|
land_num = re.search('\d+', land)
|
|
if land_num:
|
|
land = land_num.group(0)
|
|
|
|
water = str(children[10]).strip().replace(',', '')
|
|
water_num = re.search('\d+', water)
|
|
if water_num:
|
|
water = water_num.group(0)
|
|
|
|
if state == 'Mali': print(total)
|
|
states.append({'state': state, 'total': total, 'land': land, 'water': water})
|
|
else:
|
|
# print('area leaving out', children)
|
|
pass
|
|
|
|
print('area', len(states), 'items')
|
|
return states
|
|
|
|
def population_fb2000():
|
|
file = 'data/factbook/2000/population.html'
|
|
html = open(file).read()
|
|
if not html:
|
|
print('error reading file at <%s>' % file)
|
|
return False
|
|
states = []
|
|
soup = BeautifulSoup(html, 'html5lib')
|
|
for p in soup.find_all('p'):
|
|
children = list(p.children)
|
|
|
|
if len(children) >= 4:
|
|
state = children[0].text[:-1]
|
|
rate = str(children[3]).strip().replace(',', '').split(' ')[0]
|
|
states.append({'state': state, 'population': rate})
|
|
else:
|
|
# print('population leave out', children)
|
|
pass
|
|
|
|
print('population', len(states), 'items')
|
|
return states
|
|
|
|
def deathrate_fb2000():
|
|
file = 'data/factbook/2000/death_rate.html'
|
|
html = open(file).read()
|
|
if not html:
|
|
print('error reading file at <%s>' % file)
|
|
return False
|
|
states = []
|
|
soup = BeautifulSoup(html, 'html5lib')
|
|
for p in soup.find_all('p'):
|
|
children = list(p.children)
|
|
# print(children)
|
|
if len(children) >= 4:
|
|
state = children[0].text[:-1]
|
|
rate = str(children[3]).strip().replace(',', '').split(' ')[0]
|
|
states.append({'state': state, 'deathrate': rate})
|
|
|
|
print('deathrate', len(states), 'items')
|
|
return states
|
|
|
|
def birthrate_fb2000():
|
|
html = open('data/factbook/2000/birth_rate.html').read()
|
|
if not html:
|
|
print('error reading birth_rate.html')
|
|
return False
|
|
states = []
|
|
soup = BeautifulSoup(html, 'html5lib')
|
|
for p in soup.find_all('p'):
|
|
children = list(p.children)
|
|
# print(children)
|
|
if len(children) >= 4:
|
|
state = children[0].text[:-1]
|
|
rate = str(children[3]).strip().replace(',', '').split(' ')[0]
|
|
states.append({'state': state, 'birthrate': rate})
|
|
print('birthrate', len(states), 'items')
|
|
return states
|
|
|
|
def unemploymentrate_fb2000():
|
|
file = 'data/factbook/2000/unemployment_rate.html'
|
|
html = open(file).read()
|
|
if not html:
|
|
print('error reading file at <%s>' % file)
|
|
return False
|
|
states = []
|
|
soup = BeautifulSoup(html, 'html5lib')
|
|
for p in soup.find_all('p'):
|
|
children = list(p.children)
|
|
# print(children)
|
|
if len(children) >= 4:
|
|
state = children[0].text[:-1]
|
|
# print(state)
|
|
rate = str(children[3]).strip().replace(',', '').split(' ')[0]
|
|
# print(rate)
|
|
states.append({'state': state, 'unemplymentrate': rate})
|
|
print('unemplymentrate', len(states), 'items')
|
|
return states
|
|
|
|
def total_fertility_rate_fb2000():
|
|
file = 'data/factbook/2000/total_fertility_rate.html'
|
|
html = open(file).read()
|
|
if not html:
|
|
print('error reading file at <%s>' % file)
|
|
return False
|
|
states = []
|
|
soup = BeautifulSoup(html, 'html5lib')
|
|
for p in soup.find_all('p'):
|
|
children = list(p.children)
|
|
# print(children)
|
|
if len(children) >= 4:
|
|
state = children[0].text[:-1]
|
|
# print(state)
|
|
rate = str(children[3]).strip().replace(',', '').split(' ')[0]
|
|
# print(rate)
|
|
states.append({'state': state, 'fertilityrate': rate})
|
|
print('fertilityrate', len(states), 'items')
|
|
return states
|
|
|
|
def population_below_poverty_line_fb2000():
|
|
file = 'data/factbook/2000/population_below_poverty_line.html'
|
|
html = open(file).read()
|
|
if not html:
|
|
print('error reading file at <%s>' % file)
|
|
return False
|
|
states = []
|
|
soup = BeautifulSoup(html, 'html5lib')
|
|
for p in soup.find_all('p'):
|
|
children = list(p.children)
|
|
# print(children)
|
|
if len(children) >= 4:
|
|
state = children[0].text[:-1]
|
|
# print(state)
|
|
rate = str(children[3]).strip().replace(',', '').split(' ')[0]
|
|
# print(rate)
|
|
states.append({'state': state, 'rate': rate})
|
|
print('population_below_poverty_line', len(states), 'items')
|
|
return states
|
|
|
|
def life_expectancy_at_birth_fb2000():
|
|
file = 'data/factbook/2000/life_expectancy_at_birth.html'
|
|
html = open(file).read()
|
|
if not html:
|
|
print('error reading file at <%s>' % file)
|
|
return False
|
|
states = []
|
|
soup = BeautifulSoup(html, 'html5lib')
|
|
for p in soup.find_all('p'):
|
|
children = list(p.children)
|
|
# print(children)
|
|
if len(children) >= 11:
|
|
state = children[0].text[:-1]
|
|
# print(state)
|
|
total = str(children[4]).strip().replace(',', '').split(' ')[0]
|
|
# print(total)
|
|
male = str(children[7]).strip().replace(',', '').split(' ')[0]
|
|
# print(male)
|
|
female = str(children[10]).strip().replace(',', '').split(' ')[0]
|
|
# print(female)
|
|
states.append({'state': state, 'total': total, 'male': male, 'female': female})
|
|
print('life_expectancy_at_birth', len(states), 'items')
|
|
return states
|
|
|
|
def infant_mortality_rate_fb2000():
|
|
file = 'data/factbook/2000/infant_mortality_rate.html'
|
|
html = open(file).read()
|
|
if not html:
|
|
print('error reading file at <%s>' % file)
|
|
return False
|
|
states = []
|
|
soup = BeautifulSoup(html, 'html5lib')
|
|
for p in soup.find_all('p'):
|
|
children = list(p.children)
|
|
# print(children)
|
|
if len(children) >= 4:
|
|
state = children[0].text[:-1]
|
|
# print(state)
|
|
rate = str(children[3]).strip().replace(',', '').split(' ')[0]
|
|
# print(rate)
|
|
states.append({'state': state, 'rate': rate})
|
|
print('infant_mortality_rate', len(states), 'items')
|
|
return states
|
|
|
|
def gen_fb2000():
|
|
infant_mortality_rate = infant_mortality_rate_fb2000()
|
|
life_expectancy_at_birth = life_expectancy_at_birth_fb2000()
|
|
population_below_poverty_line = population_below_poverty_line_fb2000()
|
|
fertility = total_fertility_rate_fb2000()
|
|
unemployment = unemploymentrate_fb2000()
|
|
population = population_fb2000()
|
|
deathrate = deathrate_fb2000()
|
|
birthrate = birthrate_fb2000()
|
|
area = area_fb2000()
|
|
merged = []
|
|
|
|
for fert_item in fertility:
|
|
d = {'state': fert_item['state']}
|
|
d['children_born_per_woman'] = fert_item['fertilityrate']
|
|
|
|
for item in unemployment:
|
|
if fert_item['state'] == item['state']:
|
|
d['unemployment_rate'] = item['unemplymentrate']
|
|
|
|
for item in population:
|
|
if fert_item['state'] == item['state']:
|
|
d['population'] = item['population']
|
|
|
|
for item in deathrate:
|
|
if fert_item['state'] == item['state']:
|
|
d['deaths_per_1000'] = item['deathrate']
|
|
|
|
for item in birthrate:
|
|
if fert_item['state'] == item['state']:
|
|
d['births_per_1000'] = item['birthrate']
|
|
|
|
for item in area:
|
|
if fert_item['state'] == item['state']:
|
|
d['area_total_km2'] = item['total']
|
|
d['area_land_km2'] = item['land']
|
|
d['area_water_km2'] = item['water']
|
|
|
|
for item in infant_mortality_rate:
|
|
if fert_item['state'] == item['state']:
|
|
d['infant_mortality_rate'] = item['rate']
|
|
|
|
for item in population_below_poverty_line:
|
|
if fert_item['state'] == item['state']:
|
|
d['population_below_poverty_line'] = item['rate']
|
|
|
|
for item in life_expectancy_at_birth:
|
|
if fert_item['state'] == item['state']:
|
|
d['life_expectancy_at_birth_total'] = item['total']
|
|
d['life_expectancy_at_birth_male'] = item['male']
|
|
d['life_expectancy_at_birth_female'] = item['female']
|
|
|
|
merged.append(d)
|
|
|
|
# normalize names
|
|
for item in merged:
|
|
if item['state'] == 'Korea, North': item['state'] = 'North Korea'
|
|
if item['state'] == 'Korea, South': item['state'] = 'South Korea'
|
|
if item['state'] == 'Bahamas, The': item['state'] = 'The Bahamas'
|
|
if item['state'] == 'Cote d\'Ivoire': item['state'] = 'Ivory Coast'
|
|
if item['state'] == 'Congo, Republic of the': item['state'] = 'Republic of the Congo'
|
|
if item['state'] == 'Congo, Democratic Republic of the': item['state'] = 'Democratic Republic of the Congo'
|
|
if item['state'] == 'Falkland Islands (Islas Malvinas)': item['state'] = 'Falkland Islands'
|
|
if item['state'] == 'Gambia, The': item['state'] = 'Gambia'
|
|
if item['state'] == 'Macedonia, The Former Yugoslav Republic of': item['state'] = 'Macedonia'
|
|
if item['state'] == 'Serbia and Montenegro': item['state'] = 'Republic of Serbia'
|
|
|
|
print('fb2000', len(merged), 'items')
|
|
return merged
|
|
|
|
|
|
def add_fb2000_data_to_countries_geojson():
|
|
|
|
datafile = 'fb2000.json'
|
|
data = json.loads(open(datafile).read())
|
|
if not data:
|
|
print('error loading data file')
|
|
return False
|
|
|
|
countriesfile = 'data/world/countries.geo.json'
|
|
# {"type":"Feature","id":"BHS","properties":{"name":"The Bahamas"},"geometry":{"type":"MultiPolygon","coordinates":[[[[-77.53466,23.75975],[-77.78,23.71],[-78.03405,24.28615],[-78.40848,24.57564],[-78.19087,25.2103],[-77.89,25.17],[-77.54,24.34],[-77.53466,23.75975]]],[[[-77.82,26.58],[-78.91,26.42],[-78.98,26.79],[-78.51,26.87],[-77.85,26.84],[-77.82,26.58]]],[[[-77,26.59],[-77.17255,25.87918],[-77.35641,26.00735],[-77.34,26.53],[-77.78802,26.92516],[-77.79,27.04],[-77,26.59]]]]}},
|
|
|
|
countries = json.loads(open(countriesfile).read())
|
|
if not countries:
|
|
print('error loading countries file')
|
|
return False
|
|
|
|
if not countries.get('features', None) or isinstance(countries.get('features'), list) != True:
|
|
print('error reading features')
|
|
return False
|
|
|
|
for country in countries['features']:
|
|
state = country['properties']['name']
|
|
found = False
|
|
|
|
for item in data:
|
|
# print(item)
|
|
if not item.get('state', None):
|
|
print('error: no state in item')
|
|
continue
|
|
|
|
if not item.get('population', None):
|
|
print('error: no population data for <%s>' % item['state'])
|
|
continue
|
|
|
|
if not item.get('area_total_km2', None):
|
|
# print('error: no area_total_km2 data for <%s>' % item['state'])
|
|
continue
|
|
|
|
# if state == 'Greenland':
|
|
# print(item['state'])
|
|
|
|
if item['state'] in state:
|
|
|
|
density = round(int(item['population']) / int(item['area_total_km2']), 1)
|
|
# print(state, int(item['population']), item['area_total_km2'], int(item['area_total_km2']), density)
|
|
|
|
if not isinstance(density, float):
|
|
density = 'NA'
|
|
|
|
country['properties']['density'] = density
|
|
country['properties']['population'] = int(item['population'])
|
|
country['properties']['area_total_km2'] = int(item['area_total_km2'])
|
|
|
|
found = True
|
|
|
|
if not found:
|
|
print('no match found for <%s>' % state)
|
|
|
|
writejsoncompact('data/world/countries-dev.js', countries)
|
|
|
|
c = open('data/world/countries-dev.js').read()
|
|
cc = 'var statesData = ' + c
|
|
with open('data/world/countries-dev.js', 'w') as js:
|
|
js.write(cc)
|
|
|
|
def json_to_js(filein, fileout):
|
|
c = open(filein).read()
|
|
cc = 'var statesData = ' + c
|
|
with open(fileout, 'w') as js:
|
|
js.write(cc)
|
|
|
|
|
|
def eng():
|
|
|
|
geoitems = readjson('eng/regions-ons.json')
|
|
dataitems = readjson('eng/regions-wiki.json')
|
|
geoitems_namekey = 'rgn16nm'
|
|
dataitems_namekey = 'name'
|
|
|
|
for geoitem in geoitems['features']:
|
|
geoitemname = geoitem['properties'][geoitems_namekey]
|
|
match = False
|
|
|
|
for dataitem in dataitems:
|
|
dataitemname = dataitem[dataitems_namekey]
|
|
# print(dataitemname)
|
|
if dataitemname.lower() == geoitemname.lower():
|
|
match = True
|
|
pop = dataitem.get('population', 0)
|
|
area = dataitem.get('area_km2', 0)
|
|
if pop == 0 or area == 0:
|
|
density = 'NA'
|
|
else:
|
|
density = round(int(dataitem['population']) / float(dataitem['area_km2']), 1)
|
|
geoitem['properties']['name'] = geoitemname
|
|
geoitem['properties']['density'] = density
|
|
geoitem['properties']['population'] = dataitem['population']
|
|
geoitem['properties']['area_km2'] = dataitem['area_km2']
|
|
|
|
if not match:
|
|
print('no match for <%s>' % geoitemname)
|
|
|
|
writejsoncompact('eng/regions.geojson', geoitems)
|
|
json_to_js('eng/regions.geojson', 'eng/regions.js')
|
|
|
|
def nld():
|
|
|
|
provinces = readjson('nld/provinces.json')
|
|
geojson = readjson('nld/provinces.geojson')
|
|
|
|
for geoitem in geojson['features']:
|
|
|
|
name = geoitem['properties']['name'].split(' ')[0]
|
|
match = False
|
|
|
|
for item in provinces:
|
|
|
|
if name == item['name_nl']:
|
|
match = True
|
|
pop = item.get('population', 0)
|
|
area = item.get('area_km2', 0)
|
|
if pop == 0 or area == 0:
|
|
density = 'NA'
|
|
else:
|
|
density = round(int(item['population']) / float(item['area_km2']), 1)
|
|
geoitem['properties']['density'] = density
|
|
|
|
if not match:
|
|
print('no match for <%s>' % name)
|
|
|
|
writejsoncompact('netherlands/provinces-edit.geojson', geojson)
|
|
json_to_js('netherlands/provinces-edit.geojson', 'netherlands/provinces-edit.js')
|
|
|
|
def main():
|
|
|
|
# eng()
|
|
|
|
# nld()
|
|
|
|
|
|
writejson('fb2000.json', gen_fb2000())
|
|
# return
|
|
add_fb2000_data_to_countries_geojson()
|
|
return
|
|
|
|
# writejson('area-fb2000.json', area_fb2000())
|
|
# return
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|