#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys, time, codecs, re
import json
import yaml
import requests
from bs4 import BeautifulSoup
from bs4 import element
from wikitools import wiki
from wikitools import category
from wikitools import api
from wikitools import page

# server write utf-8
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

wiki_base_url = 'http://wiki.snom.com'
wiki_api_url = wiki_base_url + '/wiki/api.php'


def readlistofstates():
    # table xpath sample
    # //*[@id="mw-content-text"]/div/table[1]/tbody/tr[4]/td[1]/b/a
    url = 'https://en.wikipedia.org/wiki/List_of_sovereign_states'
    request = requests.get(url)
    if not request:
        print 'error: could not read contents from <%s>' % url
        return False
    soup = BeautifulSoup(request.text, 'html5lib')
    table = soup.find("table", class_="wikitable")
    print table
    # for tag in soup.find_all(re.compile("^dt|dd")):
    #     if tag.get_text(strip=True) != '':
    #         result_list.append(tag)
    # return result_list
    


def get_settings_for_category_in_wiki(wiki_category,wiki_api_url):
    urltitles = []
    site = wiki.Wiki(wiki_api_url) 
    cat = category.Category(site, wiki_category)
    for member in cat.getAllMembers():
        if re.match('^Setting.*',member.urltitle):
            urltitles.append(member.urltitle)
    return urltitles

# https://en.wikipedia.org/wiki/List_of_countries_and_territories_by_population_density

def getPageFromWikipedia(title):
    site = wiki.Wiki("https://en.wikipedia.org/w/api.php") 
    pageobj = page.Page(site, title)
    print pageobj.getWikiText()

def main():
    readlistofstates()
    return
    getPageFromWikipedia('List_of_countries_and_territories_by_population_density')
    return 0
    # create a Wiki object
    site = wiki.Wiki("https://en.wikipedia.org/w/api.php") 
    # login - required for read-restricted wikis
    # site.login("username", "password")
    # define the params for the query
    params = {'action':'query', 'titles':'List_of_countries_and_territories_by_population_density'}
    # create the request object
    request = api.APIRequest(site, params)
    # query the API
    result = request.query()
    print result

if __name__ == '__main__':
    sys.exit(main())