#!/usr/bin/env python # -*- coding: utf-8 -*- import sys, time, codecs, re import json import yaml import requests from bs4 import BeautifulSoup from bs4 import element from wikitools import wiki from wikitools import category # server write utf-8 import sys reload(sys) sys.setdefaultencoding('utf-8') wiki_base_url = 'http://wiki.snom.com' wiki_api_url = wiki_base_url + '/wiki/api.php' def main(): # download_all_setting_pages() # return # process_single_file() # return alist = [] alist.append({'wiki_category':'Setting:V8','output_filename':'settings-v8'}) alist.append({'wiki_category':'Setting:snom_MP','output_filename':'settings-mp'}) alist.append({'wiki_category':'Setting:snom8xx','output_filename':'settings-8xx'}) alist.append({'wiki_category':'Setting:snom3x0','output_filename':'settings-3x0'}) alist.append({'wiki_category':'Setting:Directory:LDAP','output_filename':'settings-directory-ldap'}) alist.append({'wiki_category':'Setting:Require_Reboot','output_filename':'settings-require-reboot'}) alist.append({'wiki_category':'Setting:ProvisioningOnly','output_filename':'settings-prov-only'}) for ad in alist: generate_from_pages(ad['output_filename']) return # for ad in alist: # wiki_category = ad['wiki_category'] # output_filename = ad['output_filename'] # url_titles = get_settings_for_category_in_wiki(wiki_category,wiki_api_url) # write_txt(output_filename, url_titles) # continue # write_md(output_filename, url_titles, wiki_category) # generate_from_pages() # # continue # process_num_max = 20000 # count = 0 # objs = [] # for urltitle in url_titles: # if count < process_num_max: # a_url = wiki_base_url + '/' + urltitle # key_name = extract_key_name_from_url_title(urltitle) # request = requests.get(a_url) # dt_dd_tags = list_of_dt_and_dd(request.text) # desired_obj = parse_tags(dt_dd_tags) # desired_obj['akey'] = key_name # desired_obj['url'] = a_url # objs.append(desired_obj) # count+=1 # writetojson(output_filename + '.json',objs) # writetoyaml(output_filename + '.yaml',objs) # return def generate_from_pages(filename): objs = [] with open(filename + '.txt','r') as stream: for line in stream: key_name = extract_key_name_from_url_title(line) html_str = open('settings/'+key_name+'.html','r').read() dt_dd_tags = list_of_dt_and_dd(html_str) desired_obj = parse_tags(dt_dd_tags) desired_obj['akey'] = key_name objs.append(desired_obj) writetojson(filename + '.json',objs) writetoyaml(filename + '.yaml',objs) def list_of_dt_and_dd(stream): result_list= [] soup = BeautifulSoup(stream) for tag in soup.find_all(re.compile("^dt|dd")): if tag.get_text(strip=True) != '': result_list.append(tag) return result_list # print(tag.name + ' : ' + tag.get_text(strip=True)) def get_clean_string(dirty_str): dstr = dirty_str.strip() if dstr == '.' or dstr == ',' or dstr == '' or dstr == '-' or dstr == '_' or dstr == '|' or dstr == ':' or dstr == ';': return '' else: return dstr def get_content_list(tag,include_urls=True): a_list = [] c = '' u = '' if len(tag.contents) > 1: for child in tag.contents: c = '' u = '' if isinstance(child,element.NavigableString): c = get_clean_string(child) elif isinstance(child,element.Tag): if child.name == 'a': u = ' (' + wiki_base_url + child.get('href') + ')' c = get_clean_string(child.get_text(strip=True)) if c != '': if include_urls: a_list.append(c + u) else: a_list.append(c) else: if tag.name == 'a': u = ' (' + wiki_base_url + tag.get('href') + ')' c = get_clean_string(tag.get_text(strip=True)) if c != '': if include_urls: a_list.append(c + u) else: a_list.append(c) return a_list def get_content_str(tag): cc = '' c = '' if len(tag.contents) > 1: for child in tag.contents: c = '' if isinstance(child,element.NavigableString): c = get_clean_string(child) elif isinstance(child,element.Tag): c = get_clean_string(child.get_text(strip=True)) if c != '': cc = cc + c else: c = get_clean_string(tag.get_text(strip=True)) if c != '': cc = cc + c return cc def parse_tags(listoftags): cur_key = '' fw = [] xml = '' valid = [] default = [] desc = [] for tag in listoftags: if tag.name == 'dt': # print 'key : ' + tag.get_text(strip=True) cur_key = tag.get_text(strip=True) if tag.name == 'dd': if tag.get_text(strip=True) != '': # print 'val : ' + tag.get_text(strip=True) if cur_key == 'FIRMWARE VERSIONS': fw.extend(get_content_list(tag,include_urls=False)) elif cur_key == 'XML CONFIGURATION': ll = get_content_list(tag,include_urls=False) xml = get_content_str(tag) # str(ll).strip('[]') # print xml elif cur_key == 'DESCRIPTION': desc.extend(get_content_list(tag)) elif cur_key == 'VALIDVALUE': clist = get_content_list(tag) valid.extend(clist) # split comma separated strings and make a list # for cstr in clist: # slist = cstr.split(',') # if len(slist) > 1: # for sstr in slist: # if sstr.strip() != '': # valid.append(sstr) # else: # if slist[0].strip() != '': # valid.append(slist[0]) elif cur_key == 'DEFAULTVALUE': default.extend(get_content_list(tag,include_urls=False)) return { 'fw' : fw, 'desc' : desc , 'valid' : valid , 'default' : default , 'xml' : xml } def writetojson(filename,aobj): with open(filename, 'w') as outfile: json.dump(aobj, outfile, sort_keys=True, indent=4, separators=(', ', ': ')) def writetoyaml(filename,aobj): ff = open(filename, 'wb') yaml.safe_dump(aobj, ff, default_flow_style=False) def write_md(filename,urltitles, title): s = '##### ' + title + ' (' + str(len(urltitles)) + ') ' + '\n\n' for title in urltitles: a_url = wiki_base_url + '/' + title if len(title.split('/')) > 1: key_name = title.split('/')[1] mdlinkstr = "["+ key_name +"]("+ a_url +")" s += '- ' + mdlinkstr + '\n' writeto(filename + '.md',s) def write_txt(filename,urltitles): s = '' for title in urltitles: # if len(title.split('/')) > 1: # key_name = title.split('/')[1] # s += key_name + '\n' s += title.strip() + '\n' writeto(filename + '.txt',s) def writeto(filename,astring): p = codecs.open(filename , "w" , "utf-8") p.write(astring) p.close() def listtofile(filename,listofstrings): p = codecs.open(filename , "w" , "utf-8") p.write('\n'.join(listofstrings)) p.close() def process_single_file(): with open('call_screen_fkeys.html', 'r') as stream: dt_dd_tags = list_of_dt_and_dd(stream) desired_obj = parse_tags(dt_dd_tags) desired_obj['key'] = 'call_screen_fkeys' desired_obj['url'] = 'http://wiki.snom.com/Settings/call_screen_fkeys' print desired_obj def get_next_non_empty_dd(node): a_value = '' if node.next_sibling: if node.next_sibling.name == 'dd': a_value = node.next_sibling.get_text(strip=True) if node.next_sibling.next_sibling: if node.next_sibling.next_sibling.name == 'dd': a_value = node.next_sibling.next_sibling.get_text(strip=True) elif node.next_sibling.next_sibling.name == 'dt': a_value = node.next_sibling.get_text(strip=True) else: a_value = node.next_sibling.get_text(strip=True) return unicode(a_value) def get_settings_for_category_in_wiki(wiki_category,wiki_api_url): urltitles = [] site = wiki.Wiki(wiki_api_url) cat = category.Category(site, wiki_category) for member in cat.getAllMembers(): if re.match('^Setting.*',member.urltitle): urltitles.append(member.urltitle) return urltitles def download_all_setting_pages(): # settings.txt contains any urltitle to settings wiki page # create with # alist = [] # alist.append({'wiki_category':'Setting:V8','output_filename':'settings-v8'}) # alist.append({'wiki_category':'Setting:snom_MP','output_filename':'settings-mp'}) # alist.append({'wiki_category':'Setting:snom8xx','output_filename':'settings-8xx'}) # alist.append({'wiki_category':'Setting:snom3x0','output_filename':'settings-3x0'}) # alist.append({'wiki_category':'Setting:Directory:LDAP','output_filename':'settings-directory-ldap'}) # alist.append({'wiki_category':'Setting:Require_Reboot','output_filename':'settings-require-reboot'}) # alist.append({'wiki_category':'Setting:ProvisioningOnly','output_filename':'settings-prov-only'}) # for ad in alist: # wiki_category = ad['wiki_category'] # output_filename = ad['output_filename'] # url_titles = get_settings_for_category_in_wiki(wiki_category,wiki_api_url) # write_txt(output_filename, url_titles) # cat *.txt | sort -u > settings.txt with open('settings.txt','r') as stream: if stream: for line in stream: a_url = wiki_base_url + '/' + line.strip() # print a_url # continue request = requests.get(a_url) if request: title = line.split('/') title_len = len(title) if title_len == 2: key_name = title[1] elif title_len == 3: key_name = title[1] + '_' + title[2] writeto('settings/'+key_name.strip()+'.html',request.text) def extract_key_name_from_url_title(urltitle): tit = urltitle.split('/') titlen = len(tit) if titlen == 2: key_name = tit[1].strip() elif titlen == 3: key_name = tit[1].strip() + '_' + tit[2].strip() return key_name main() sys.exit(0)