354 lines
9.3 KiB
Python
354 lines
9.3 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import sys, time, codecs, re
|
|
import json
|
|
import yaml
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from bs4 import element
|
|
from wikitools import wiki
|
|
from wikitools import category
|
|
|
|
# server write utf-8
|
|
import sys
|
|
reload(sys)
|
|
sys.setdefaultencoding('utf-8')
|
|
|
|
wiki_base_url = 'http://wiki.snom.com'
|
|
wiki_api_url = wiki_base_url + '/wiki/api.php'
|
|
|
|
|
|
def main():
|
|
|
|
# download_all_setting_pages()
|
|
# return
|
|
# process_single_file()
|
|
# return
|
|
|
|
alist = []
|
|
|
|
alist.append({'wiki_category':'Setting:V8','output_filename':'settings-v8'})
|
|
alist.append({'wiki_category':'Setting:snom_MP','output_filename':'settings-mp'})
|
|
alist.append({'wiki_category':'Setting:snom8xx','output_filename':'settings-8xx'})
|
|
alist.append({'wiki_category':'Setting:snom3x0','output_filename':'settings-3x0'})
|
|
alist.append({'wiki_category':'Setting:Directory:LDAP','output_filename':'settings-directory-ldap'})
|
|
alist.append({'wiki_category':'Setting:Require_Reboot','output_filename':'settings-require-reboot'})
|
|
alist.append({'wiki_category':'Setting:ProvisioningOnly','output_filename':'settings-prov-only'})
|
|
|
|
for ad in alist:
|
|
generate_from_pages(ad['output_filename'])
|
|
|
|
return
|
|
|
|
# for ad in alist:
|
|
|
|
# wiki_category = ad['wiki_category']
|
|
# output_filename = ad['output_filename']
|
|
|
|
# url_titles = get_settings_for_category_in_wiki(wiki_category,wiki_api_url)
|
|
|
|
# write_txt(output_filename, url_titles)
|
|
|
|
# continue
|
|
|
|
# write_md(output_filename, url_titles, wiki_category)
|
|
|
|
|
|
# generate_from_pages()
|
|
# # continue
|
|
|
|
# process_num_max = 20000
|
|
# count = 0
|
|
# objs = []
|
|
# for urltitle in url_titles:
|
|
# if count < process_num_max:
|
|
# a_url = wiki_base_url + '/' + urltitle
|
|
# key_name = extract_key_name_from_url_title(urltitle)
|
|
# request = requests.get(a_url)
|
|
# dt_dd_tags = list_of_dt_and_dd(request.text)
|
|
|
|
# desired_obj = parse_tags(dt_dd_tags)
|
|
# desired_obj['akey'] = key_name
|
|
# desired_obj['url'] = a_url
|
|
|
|
# objs.append(desired_obj)
|
|
|
|
# count+=1
|
|
|
|
# writetojson(output_filename + '.json',objs)
|
|
# writetoyaml(output_filename + '.yaml',objs)
|
|
|
|
# return
|
|
|
|
def generate_from_pages(filename):
|
|
objs = []
|
|
with open(filename + '.txt','r') as stream:
|
|
for line in stream:
|
|
key_name = extract_key_name_from_url_title(line)
|
|
html_str = open('settings/'+key_name+'.html','r').read()
|
|
dt_dd_tags = list_of_dt_and_dd(html_str)
|
|
desired_obj = parse_tags(dt_dd_tags)
|
|
desired_obj['akey'] = key_name
|
|
objs.append(desired_obj)
|
|
|
|
writetojson(filename + '.json',objs)
|
|
writetoyaml(filename + '.yaml',objs)
|
|
|
|
def list_of_dt_and_dd(stream):
|
|
result_list= []
|
|
soup = BeautifulSoup(stream)
|
|
for tag in soup.find_all(re.compile("^dt|dd")):
|
|
if tag.get_text(strip=True) != '':
|
|
result_list.append(tag)
|
|
return result_list
|
|
# print(tag.name + ' : ' + tag.get_text(strip=True))
|
|
|
|
def get_clean_string(dirty_str):
|
|
dstr = dirty_str.strip()
|
|
if dstr == '.' or dstr == ',' or dstr == '' or dstr == '-' or dstr == '_' or dstr == '|' or dstr == ':' or dstr == ';':
|
|
return ''
|
|
else:
|
|
return dstr
|
|
|
|
def get_content_list(tag,include_urls=True):
|
|
a_list = []
|
|
c = ''
|
|
u = ''
|
|
if len(tag.contents) > 1:
|
|
for child in tag.contents:
|
|
c = ''
|
|
u = ''
|
|
if isinstance(child,element.NavigableString):
|
|
c = get_clean_string(child)
|
|
elif isinstance(child,element.Tag):
|
|
if child.name == 'a':
|
|
u = ' (' + wiki_base_url + child.get('href') + ')'
|
|
c = get_clean_string(child.get_text(strip=True))
|
|
if c != '':
|
|
if include_urls:
|
|
a_list.append(c + u)
|
|
else:
|
|
a_list.append(c)
|
|
else:
|
|
if tag.name == 'a':
|
|
u = ' (' + wiki_base_url + tag.get('href') + ')'
|
|
c = get_clean_string(tag.get_text(strip=True))
|
|
if c != '':
|
|
if include_urls:
|
|
a_list.append(c + u)
|
|
else:
|
|
a_list.append(c)
|
|
|
|
return a_list
|
|
|
|
def get_content_str(tag):
|
|
cc = ''
|
|
c = ''
|
|
|
|
if len(tag.contents) > 1:
|
|
for child in tag.contents:
|
|
c = ''
|
|
if isinstance(child,element.NavigableString):
|
|
c = get_clean_string(child)
|
|
elif isinstance(child,element.Tag):
|
|
c = get_clean_string(child.get_text(strip=True))
|
|
if c != '':
|
|
cc = cc + c
|
|
else:
|
|
c = get_clean_string(tag.get_text(strip=True))
|
|
if c != '':
|
|
cc = cc + c
|
|
|
|
return cc
|
|
|
|
def parse_tags(listoftags):
|
|
|
|
cur_key = ''
|
|
fw = []
|
|
xml = ''
|
|
valid = []
|
|
default = []
|
|
desc = []
|
|
|
|
for tag in listoftags:
|
|
|
|
if tag.name == 'dt':
|
|
# print 'key : ' + tag.get_text(strip=True)
|
|
cur_key = tag.get_text(strip=True)
|
|
|
|
if tag.name == 'dd':
|
|
|
|
if tag.get_text(strip=True) != '':
|
|
|
|
# print 'val : ' + tag.get_text(strip=True)
|
|
|
|
if cur_key == 'FIRMWARE VERSIONS':
|
|
fw.extend(get_content_list(tag,include_urls=False))
|
|
|
|
elif cur_key == 'XML CONFIGURATION':
|
|
ll = get_content_list(tag,include_urls=False)
|
|
xml = get_content_str(tag) # str(ll).strip('[]')
|
|
# print xml
|
|
|
|
elif cur_key == 'DESCRIPTION':
|
|
desc.extend(get_content_list(tag))
|
|
|
|
elif cur_key == 'VALIDVALUE':
|
|
clist = get_content_list(tag)
|
|
valid.extend(clist)
|
|
# split comma separated strings and make a list
|
|
# for cstr in clist:
|
|
# slist = cstr.split(',')
|
|
# if len(slist) > 1:
|
|
# for sstr in slist:
|
|
# if sstr.strip() != '':
|
|
# valid.append(sstr)
|
|
# else:
|
|
# if slist[0].strip() != '':
|
|
# valid.append(slist[0])
|
|
|
|
|
|
|
|
|
|
elif cur_key == 'DEFAULTVALUE':
|
|
default.extend(get_content_list(tag,include_urls=False))
|
|
|
|
return { 'fw' : fw, 'desc' : desc , 'valid' : valid , 'default' : default , 'xml' : xml }
|
|
|
|
def writetojson(filename,aobj):
|
|
with open(filename, 'w') as outfile:
|
|
json.dump(aobj, outfile, sort_keys=True, indent=4, separators=(', ', ': '))
|
|
|
|
def writetoyaml(filename,aobj):
|
|
ff = open(filename, 'wb')
|
|
yaml.safe_dump(aobj, ff, default_flow_style=False)
|
|
|
|
|
|
def write_md(filename,urltitles, title):
|
|
s = '##### ' + title + ' (' + str(len(urltitles)) + ') ' + '\n\n'
|
|
for title in urltitles:
|
|
a_url = wiki_base_url + '/' + title
|
|
if len(title.split('/')) > 1:
|
|
key_name = title.split('/')[1]
|
|
mdlinkstr = "["+ key_name +"]("+ a_url +")"
|
|
s += '- ' + mdlinkstr + '\n'
|
|
writeto(filename + '.md',s)
|
|
|
|
def write_txt(filename,urltitles):
|
|
s = ''
|
|
for title in urltitles:
|
|
# if len(title.split('/')) > 1:
|
|
# key_name = title.split('/')[1]
|
|
# s += key_name + '\n'
|
|
s += title.strip() + '\n'
|
|
writeto(filename + '.txt',s)
|
|
|
|
def writeto(filename,astring):
|
|
p = codecs.open(filename , "w" , "utf-8")
|
|
p.write(astring)
|
|
p.close()
|
|
|
|
def listtofile(filename,listofstrings):
|
|
p = codecs.open(filename , "w" , "utf-8")
|
|
p.write('\n'.join(listofstrings))
|
|
p.close()
|
|
|
|
def process_single_file():
|
|
|
|
with open('call_screen_fkeys.html', 'r') as stream:
|
|
|
|
dt_dd_tags = list_of_dt_and_dd(stream)
|
|
|
|
desired_obj = parse_tags(dt_dd_tags)
|
|
desired_obj['key'] = 'call_screen_fkeys'
|
|
desired_obj['url'] = 'http://wiki.snom.com/Settings/call_screen_fkeys'
|
|
|
|
print desired_obj
|
|
|
|
def get_next_non_empty_dd(node):
|
|
a_value = ''
|
|
if node.next_sibling:
|
|
|
|
if node.next_sibling.name == 'dd':
|
|
|
|
a_value = node.next_sibling.get_text(strip=True)
|
|
|
|
if node.next_sibling.next_sibling:
|
|
|
|
if node.next_sibling.next_sibling.name == 'dd':
|
|
a_value = node.next_sibling.next_sibling.get_text(strip=True)
|
|
|
|
elif node.next_sibling.next_sibling.name == 'dt':
|
|
a_value = node.next_sibling.get_text(strip=True)
|
|
|
|
else:
|
|
a_value = node.next_sibling.get_text(strip=True)
|
|
|
|
return unicode(a_value)
|
|
|
|
|
|
def get_settings_for_category_in_wiki(wiki_category,wiki_api_url):
|
|
urltitles = []
|
|
site = wiki.Wiki(wiki_api_url)
|
|
cat = category.Category(site, wiki_category)
|
|
for member in cat.getAllMembers():
|
|
if re.match('^Setting.*',member.urltitle):
|
|
urltitles.append(member.urltitle)
|
|
return urltitles
|
|
|
|
|
|
|
|
def download_all_setting_pages():
|
|
# settings.txt contains any urltitle to settings wiki page
|
|
# create with
|
|
# alist = []
|
|
# alist.append({'wiki_category':'Setting:V8','output_filename':'settings-v8'})
|
|
# alist.append({'wiki_category':'Setting:snom_MP','output_filename':'settings-mp'})
|
|
# alist.append({'wiki_category':'Setting:snom8xx','output_filename':'settings-8xx'})
|
|
# alist.append({'wiki_category':'Setting:snom3x0','output_filename':'settings-3x0'})
|
|
# alist.append({'wiki_category':'Setting:Directory:LDAP','output_filename':'settings-directory-ldap'})
|
|
# alist.append({'wiki_category':'Setting:Require_Reboot','output_filename':'settings-require-reboot'})
|
|
# alist.append({'wiki_category':'Setting:ProvisioningOnly','output_filename':'settings-prov-only'})
|
|
|
|
# for ad in alist:
|
|
# wiki_category = ad['wiki_category']
|
|
# output_filename = ad['output_filename']
|
|
# url_titles = get_settings_for_category_in_wiki(wiki_category,wiki_api_url)
|
|
# write_txt(output_filename, url_titles)
|
|
# cat *.txt | sort -u > settings.txt
|
|
|
|
with open('settings.txt','r') as stream:
|
|
if stream:
|
|
for line in stream:
|
|
a_url = wiki_base_url + '/' + line.strip()
|
|
# print a_url
|
|
# continue
|
|
request = requests.get(a_url)
|
|
if request:
|
|
title = line.split('/')
|
|
title_len = len(title)
|
|
if title_len == 2:
|
|
key_name = title[1]
|
|
elif title_len == 3:
|
|
key_name = title[1] + '_' + title[2]
|
|
writeto('settings/'+key_name.strip()+'.html',request.text)
|
|
|
|
def extract_key_name_from_url_title(urltitle):
|
|
tit = urltitle.split('/')
|
|
titlen = len(tit)
|
|
if titlen == 2:
|
|
key_name = tit[1].strip()
|
|
elif titlen == 3:
|
|
key_name = tit[1].strip() + '_' + tit[2].strip()
|
|
return key_name
|
|
|
|
|
|
|
|
|
|
|
|
|
|
main()
|
|
sys.exit(0)
|
|
|