#!/usr/bin/env python3 import sys, time, codecs, re import json import yaml import requests from bs4 import BeautifulSoup from bs4 import element def writejson(filename, aobj): with open(filename, 'w') as outfile: json.dump(aobj, outfile, sort_keys=True, indent=4, separators=(', ', ': ')) def writejsoncompact(filename, aobj): with open(filename, 'w') as outfile: json.dump(aobj, outfile, sort_keys=True, separators=(',', ':')) def readjson(file): print('reading json from %s ...' % file) data = json.loads(open(file).read()) if not data: print('error loading data file') return False return data def area_fb2000(): html = open('data/factbook/2000/area.html').read() if not html: print('error reading area.html') return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 11: state = children[0].text[:-1] total = str(children[4]).strip().replace(',', '') if 'million' in total: total_num = float(total.split(' ')[0]) * 1000000 total = total_num else: total_num = re.search('\d+', total) if total_num: total = total_num.group(0) land = str(children[7]).strip().replace(',', '') land_num = re.search('\d+', land) if land_num: land = land_num.group(0) water = str(children[10]).strip().replace(',', '') water_num = re.search('\d+', water) if water_num: water = water_num.group(0) if state == 'Mali': print(total) states.append({'state': state, 'total': total, 'land': land, 'water': water}) else: # print('area leaving out', children) pass print('area', len(states), 'items') return states def population_fb2000(): file = 'data/factbook/2000/population.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) if len(children) >= 4: state = children[0].text[:-1] rate = str(children[3]).strip().replace(',', '').split(' ')[0] states.append({'state': state, 'population': rate}) else: # print('population leave out', children) pass print('population', len(states), 'items') return states def deathrate_fb2000(): file = 'data/factbook/2000/death_rate.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 4: state = children[0].text[:-1] rate = str(children[3]).strip().replace(',', '').split(' ')[0] states.append({'state': state, 'deathrate': rate}) print('deathrate', len(states), 'items') return states def birthrate_fb2000(): html = open('data/factbook/2000/birth_rate.html').read() if not html: print('error reading birth_rate.html') return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 4: state = children[0].text[:-1] rate = str(children[3]).strip().replace(',', '').split(' ')[0] states.append({'state': state, 'birthrate': rate}) print('birthrate', len(states), 'items') return states def unemploymentrate_fb2000(): file = 'data/factbook/2000/unemployment_rate.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 4: state = children[0].text[:-1] # print(state) rate = str(children[3]).strip().replace(',', '').split(' ')[0] # print(rate) states.append({'state': state, 'unemplymentrate': rate}) print('unemplymentrate', len(states), 'items') return states def total_fertility_rate_fb2000(): file = 'data/factbook/2000/total_fertility_rate.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 4: state = children[0].text[:-1] # print(state) rate = str(children[3]).strip().replace(',', '').split(' ')[0] # print(rate) states.append({'state': state, 'fertilityrate': rate}) print('fertilityrate', len(states), 'items') return states def population_below_poverty_line_fb2000(): file = 'data/factbook/2000/population_below_poverty_line.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 4: state = children[0].text[:-1] # print(state) rate = str(children[3]).strip().replace(',', '').split(' ')[0] # print(rate) states.append({'state': state, 'rate': rate}) print('population_below_poverty_line', len(states), 'items') return states def life_expectancy_at_birth_fb2000(): file = 'data/factbook/2000/life_expectancy_at_birth.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 11: state = children[0].text[:-1] # print(state) total = str(children[4]).strip().replace(',', '').split(' ')[0] # print(total) male = str(children[7]).strip().replace(',', '').split(' ')[0] # print(male) female = str(children[10]).strip().replace(',', '').split(' ')[0] # print(female) states.append({'state': state, 'total': total, 'male': male, 'female': female}) print('life_expectancy_at_birth', len(states), 'items') return states def infant_mortality_rate_fb2000(): file = 'data/factbook/2000/infant_mortality_rate.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 4: state = children[0].text[:-1] # print(state) rate = str(children[3]).strip().replace(',', '').split(' ')[0] # print(rate) states.append({'state': state, 'rate': rate}) print('infant_mortality_rate', len(states), 'items') return states def gen_fb2000(): infant_mortality_rate = infant_mortality_rate_fb2000() life_expectancy_at_birth = life_expectancy_at_birth_fb2000() population_below_poverty_line = population_below_poverty_line_fb2000() fertility = total_fertility_rate_fb2000() unemployment = unemploymentrate_fb2000() population = population_fb2000() deathrate = deathrate_fb2000() birthrate = birthrate_fb2000() area = area_fb2000() merged = [] for fert_item in fertility: d = {'state': fert_item['state']} d['children_born_per_woman'] = fert_item['fertilityrate'] for item in unemployment: if fert_item['state'] == item['state']: d['unemployment_rate'] = item['unemplymentrate'] for item in population: if fert_item['state'] == item['state']: d['population'] = item['population'] for item in deathrate: if fert_item['state'] == item['state']: d['deaths_per_1000'] = item['deathrate'] for item in birthrate: if fert_item['state'] == item['state']: d['births_per_1000'] = item['birthrate'] for item in area: if fert_item['state'] == item['state']: d['area_total_km2'] = item['total'] d['area_land_km2'] = item['land'] d['area_water_km2'] = item['water'] for item in infant_mortality_rate: if fert_item['state'] == item['state']: d['infant_mortality_rate'] = item['rate'] for item in population_below_poverty_line: if fert_item['state'] == item['state']: d['population_below_poverty_line'] = item['rate'] for item in life_expectancy_at_birth: if fert_item['state'] == item['state']: d['life_expectancy_at_birth_total'] = item['total'] d['life_expectancy_at_birth_male'] = item['male'] d['life_expectancy_at_birth_female'] = item['female'] merged.append(d) # normalize names for item in merged: if item['state'] == 'Korea, North': item['state'] = 'North Korea' if item['state'] == 'Korea, South': item['state'] = 'South Korea' if item['state'] == 'Bahamas, The': item['state'] = 'The Bahamas' if item['state'] == 'Cote d\'Ivoire': item['state'] = 'Ivory Coast' if item['state'] == 'Congo, Republic of the': item['state'] = 'Republic of the Congo' if item['state'] == 'Congo, Democratic Republic of the': item['state'] = 'Democratic Republic of the Congo' if item['state'] == 'Falkland Islands (Islas Malvinas)': item['state'] = 'Falkland Islands' if item['state'] == 'Gambia, The': item['state'] = 'Gambia' if item['state'] == 'Macedonia, The Former Yugoslav Republic of': item['state'] = 'Macedonia' if item['state'] == 'Serbia and Montenegro': item['state'] = 'Republic of Serbia' print('fb2000', len(merged), 'items') return merged def add_fb2000_data_to_countries_geojson(): datafile = 'fb2000.json' data = json.loads(open(datafile).read()) if not data: print('error loading data file') return False countriesfile = 'data/world/countries.geo.json' # {"type":"Feature","id":"BHS","properties":{"name":"The Bahamas"},"geometry":{"type":"MultiPolygon","coordinates":[[[[-77.53466,23.75975],[-77.78,23.71],[-78.03405,24.28615],[-78.40848,24.57564],[-78.19087,25.2103],[-77.89,25.17],[-77.54,24.34],[-77.53466,23.75975]]],[[[-77.82,26.58],[-78.91,26.42],[-78.98,26.79],[-78.51,26.87],[-77.85,26.84],[-77.82,26.58]]],[[[-77,26.59],[-77.17255,25.87918],[-77.35641,26.00735],[-77.34,26.53],[-77.78802,26.92516],[-77.79,27.04],[-77,26.59]]]]}}, countries = json.loads(open(countriesfile).read()) if not countries: print('error loading countries file') return False if not countries.get('features', None) or isinstance(countries.get('features'), list) != True: print('error reading features') return False for country in countries['features']: state = country['properties']['name'] found = False for item in data: # print(item) if not item.get('state', None): print('error: no state in item') continue if not item.get('population', None): print('error: no population data for <%s>' % item['state']) continue if not item.get('area_total_km2', None): # print('error: no area_total_km2 data for <%s>' % item['state']) continue # if state == 'Greenland': # print(item['state']) if item['state'] in state: density = round(int(item['population']) / int(item['area_total_km2']), 1) # print(state, int(item['population']), item['area_total_km2'], int(item['area_total_km2']), density) if not isinstance(density, float): density = 'NA' country['properties']['density'] = density country['properties']['population'] = int(item['population']) country['properties']['area_total_km2'] = int(item['area_total_km2']) found = True if not found: print('no match found for <%s>' % state) writejsoncompact('data/world/countries-dev.js', countries) c = open('data/world/countries-dev.js').read() cc = 'var statesData = ' + c with open('data/world/countries-dev.js', 'w') as js: js.write(cc) def json_to_js(filein, fileout): c = open(filein).read() cc = 'var statesData = ' + c with open(fileout, 'w') as js: js.write(cc) def eng(): geoitems = readjson('eng/regions-ons.json') dataitems = readjson('eng/regions-wiki.json') geoitems_namekey = 'rgn16nm' dataitems_namekey = 'name' for geoitem in geoitems['features']: geoitemname = geoitem['properties'][geoitems_namekey] match = False for dataitem in dataitems: dataitemname = dataitem[dataitems_namekey] # print(dataitemname) if dataitemname.lower() == geoitemname.lower(): match = True pop = dataitem.get('population', 0) area = dataitem.get('area_km2', 0) if pop == 0 or area == 0: density = 'NA' else: density = round(int(dataitem['population']) / float(dataitem['area_km2']), 1) geoitem['properties']['name'] = geoitemname geoitem['properties']['density'] = density geoitem['properties']['population'] = dataitem['population'] geoitem['properties']['area_km2'] = dataitem['area_km2'] if not match: print('no match for <%s>' % geoitemname) writejsoncompact('eng/regions.geojson', geoitems) json_to_js('eng/regions.geojson', 'eng/regions.js') def nld(): provinces = readjson('nld/provinces.json') geojson = readjson('nld/provinces.geojson') for geoitem in geojson['features']: name = geoitem['properties']['name'].split(' ')[0] match = False for item in provinces: if name == item['name_nl']: match = True pop = item.get('population', 0) area = item.get('area_km2', 0) if pop == 0 or area == 0: density = 'NA' else: density = round(int(item['population']) / float(item['area_km2']), 1) geoitem['properties']['density'] = density if not match: print('no match for <%s>' % name) writejsoncompact('netherlands/provinces-edit.geojson', geojson) json_to_js('netherlands/provinces-edit.geojson', 'netherlands/provinces-edit.js') def main(): # eng() # nld() writejson('fb2000.json', gen_fb2000()) # return add_fb2000_data_to_countries_geojson() return # writejson('area-fb2000.json', area_fb2000()) # return if __name__ == '__main__': sys.exit(main())