#!/usr/bin/env python3 import sys, time, codecs, re import json import yaml import requests from bs4 import BeautifulSoup from bs4 import element def area_fb2000(): html = open('factbook2000/fields/area.html').read() if not html: print('error reading area.html') return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 11: state = children[0].text[:-1] total = str(children[4]).strip().replace(',', '') if 'million' in total: total_num = float(total.split(' ')[0]) * 1000000 total = total_num else: total_num = re.search('\d+', total) if total_num: total = total_num.group(0) land = str(children[7]).strip().replace(',', '') land_num = re.search('\d+', land) if land_num: land = land_num.group(0) water = str(children[10]).strip().replace(',', '') water_num = re.search('\d+', water) if water_num: water = water_num.group(0) if state == 'Mali': print(total) states.append({'state': state, 'total': total, 'land': land, 'water': water}) else: # print('area leaving out', children) pass print('area', len(states), 'items') return states def population_fb2000(): file = 'factbook2000/fields/population.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) if len(children) >= 4: state = children[0].text[:-1] rate = str(children[3]).strip().replace(',', '').split(' ')[0] states.append({'state': state, 'population': rate}) else: # print('population leave out', children) pass print('population', len(states), 'items') return states def deathrate_fb2000(): file = 'factbook2000/fields/death_rate.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 4: state = children[0].text[:-1] rate = str(children[3]).strip().replace(',', '').split(' ')[0] states.append({'state': state, 'deathrate': rate}) print('deathrate', len(states), 'items') return states def birthrate_fb2000(): html = open('factbook2000/fields/birth_rate.html').read() if not html: print('error reading birth_rate.html') return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 4: state = children[0].text[:-1] rate = str(children[3]).strip().replace(',', '').split(' ')[0] states.append({'state': state, 'birthrate': rate}) print('birthrate', len(states), 'items') return states def unemploymentrate_fb2000(): file = 'factbook2000/fields/unemployment_rate.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 4: state = children[0].text[:-1] # print(state) rate = str(children[3]).strip().replace(',', '').split(' ')[0] # print(rate) states.append({'state': state, 'unemplymentrate': rate}) print('unemplymentrate', len(states), 'items') return states def total_fertility_rate_fb2000(): file = 'factbook2000/fields/total_fertility_rate.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 4: state = children[0].text[:-1] # print(state) rate = str(children[3]).strip().replace(',', '').split(' ')[0] # print(rate) states.append({'state': state, 'fertilityrate': rate}) print('fertilityrate', len(states), 'items') return states def population_below_poverty_line_fb2000(): file = 'factbook2000/fields/population_below_poverty_line.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 4: state = children[0].text[:-1] # print(state) rate = str(children[3]).strip().replace(',', '').split(' ')[0] # print(rate) states.append({'state': state, 'rate': rate}) print('population_below_poverty_line', len(states), 'items') return states def life_expectancy_at_birth_fb2000(): file = 'factbook2000/fields/life_expectancy_at_birth.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 11: state = children[0].text[:-1] # print(state) total = str(children[4]).strip().replace(',', '').split(' ')[0] # print(total) male = str(children[7]).strip().replace(',', '').split(' ')[0] # print(male) female = str(children[10]).strip().replace(',', '').split(' ')[0] # print(female) states.append({'state': state, 'total': total, 'male': male, 'female': female}) print('life_expectancy_at_birth', len(states), 'items') return states def infant_mortality_rate_fb2000(): file = 'factbook2000/fields/infant_mortality_rate.html' html = open(file).read() if not html: print('error reading file at <%s>' % file) return False states = [] soup = BeautifulSoup(html, 'html5lib') for p in soup.find_all('p'): children = list(p.children) # print(children) if len(children) >= 4: state = children[0].text[:-1] # print(state) rate = str(children[3]).strip().replace(',', '').split(' ')[0] # print(rate) states.append({'state': state, 'rate': rate}) print('infant_mortality_rate', len(states), 'items') return states def gen_fb2000(): infant_mortality_rate = infant_mortality_rate_fb2000() life_expectancy_at_birth = life_expectancy_at_birth_fb2000() population_below_poverty_line = population_below_poverty_line_fb2000() fertility = total_fertility_rate_fb2000() unemployment = unemploymentrate_fb2000() population = population_fb2000() deathrate = deathrate_fb2000() birthrate = birthrate_fb2000() area = area_fb2000() merged = [] for fert_item in fertility: d = {'state': fert_item['state']} d['children_born_per_woman'] = fert_item['fertilityrate'] for item in unemployment: if fert_item['state'] == item['state']: d['unemployment_rate'] = item['unemplymentrate'] for item in population: if fert_item['state'] == item['state']: d['population'] = item['population'] for item in deathrate: if fert_item['state'] == item['state']: d['deaths_per_1000'] = item['deathrate'] for item in birthrate: if fert_item['state'] == item['state']: d['births_per_1000'] = item['birthrate'] for item in area: if fert_item['state'] == item['state']: d['area_total_km2'] = item['total'] d['area_land_km2'] = item['land'] d['area_water_km2'] = item['water'] for item in infant_mortality_rate: if fert_item['state'] == item['state']: d['infant_mortality_rate'] = item['rate'] for item in population_below_poverty_line: if fert_item['state'] == item['state']: d['population_below_poverty_line'] = item['rate'] for item in life_expectancy_at_birth: if fert_item['state'] == item['state']: d['life_expectancy_at_birth_total'] = item['total'] d['life_expectancy_at_birth_male'] = item['male'] d['life_expectancy_at_birth_female'] = item['female'] merged.append(d) # normalize names for item in merged: if item['state'] == 'Korea, North': item['state'] = 'North Korea' if item['state'] == 'Korea, South': item['state'] = 'South Korea' if item['state'] == 'Bahamas, The': item['state'] = 'The Bahamas' if item['state'] == 'Cote d\'Ivoire': item['state'] = 'Ivory Coast' if item['state'] == 'Congo, Republic of the': item['state'] = 'Republic of the Congo' if item['state'] == 'Congo, Democratic Republic of the': item['state'] = 'Democratic Republic of the Congo' if item['state'] == 'Falkland Islands (Islas Malvinas)': item['state'] = 'Falkland Islands' if item['state'] == 'Gambia, The': item['state'] = 'Gambia' if item['state'] == 'Macedonia, The Former Yugoslav Republic of': item['state'] = 'Macedonia' if item['state'] == 'Serbia and Montenegro': item['state'] = 'Republic of Serbia' print('fb2000', len(merged), 'items') return merged def writejson(filename, aobj): with open(filename, 'w') as outfile: json.dump(aobj, outfile, sort_keys=True, indent=4, separators=(', ', ': ')) def writejsoncompact(filename, aobj): with open(filename, 'w') as outfile: json.dump(aobj, outfile, sort_keys=True, separators=(',', ':')) def readjson(file): print('reading json from %s ...' % file) data = json.loads(open(file).read()) if not data: print('error loading data file') return False return data def add_fb2000_data_to_countries_geojson(): datafile = 'fb2000.json' data = json.loads(open(datafile).read()) if not data: print('error loading data file') return False countriesfile = 'countries.geo.json' countries = json.loads(open(countriesfile).read()) if not countries: print('error loading countries file') return False if not countries.get('features', None) or isinstance(countries.get('features'), list) != True: print('error reading features') return False for country in countries['features']: state = country['properties']['name'] found = False for item in data: # print(item) if not item.get('state', None): print('error: no state in item') continue if not item.get('population', None): print('error: no population data for <%s>' % item['state']) continue if not item.get('area_total_km2', None): # print('error: no area_total_km2 data for <%s>' % item['state']) continue # if state == 'Greenland': # print(item['state']) if item['state'] in state: density = round(int(item['population']) / int(item['area_total_km2']), 1) # print(state, int(item['population']), item['area_total_km2'], int(item['area_total_km2']), density) if not isinstance(density, float): density = 'NA' country['properties']['density'] = density found = True if not found: print('no match found for <%s>' % state) writejsoncompact('countries.js', countries) c = open('countries.js').read() cc = 'var statesData = ' + c with open('countries.js', 'w') as js: js.write(cc) def json_to_js(filein, fileout): c = open(filein).read() cc = 'var statesData = ' + c with open(fileout, 'w') as js: js.write(cc) def main(): geoitems = readjson('eng/regions-ons.json') dataitems = readjson('eng/regions-wiki.json') geoitems_namekey = 'rgn16nm' dataitems_namekey = 'name' for geoitem in geoitems['features']: geoitemname = geoitem['properties'][geoitems_namekey] match = False for dataitem in dataitems: dataitemname = dataitem[dataitems_namekey] # print(dataitemname) if dataitemname.lower() == geoitemname.lower(): match = True pop = dataitem.get('population', 0) area = dataitem.get('area_km2', 0) if pop == 0 or area == 0: density = 'NA' else: density = round(int(dataitem['population']) / float(dataitem['area_km2']), 1) geoitem['properties']['name'] = geoitemname geoitem['properties']['density'] = density if not match: print('no match for <%s>' % geoitemname) writejsoncompact('eng/regions.geojson', geoitems) json_to_js('eng/regions.geojson', 'eng/regions.js') return provinces = readjson('nld/provinces.json') geojson = readjson('nld/provinces.geojson') for geoitem in geojson['features']: name = geoitem['properties']['name'].split(' ')[0] match = False for item in provinces: if name == item['name_nl']: match = True pop = item.get('population', 0) area = item.get('area_km2', 0) if pop == 0 or area == 0: density = 'NA' else: density = round(int(item['population']) / float(item['area_km2']), 1) geoitem['properties']['density'] = density if not match: print('no match for <%s>' % name) writejsoncompact('netherlands/provinces-edit.geojson', geojson) json_to_js('netherlands/provinces-edit.geojson', 'netherlands/provinces-edit.js') return writejson('fb2000.json', gen_fb2000()) # return add_fb2000_data_to_countries_geojson() return # writejson('area-fb2000.json', area_fb2000()) # return if __name__ == '__main__': sys.exit(main())