Created a new module called planethandler.py. And purified planet.py a bit. Also did some optimisation about how we deliver data to DB.
This commit is contained in:
parent
d2a20f1113
commit
73e8010923
101
DJAGEN/branches/mustafa_branch/djagen/collector/planethandler.py
Executable file
101
DJAGEN/branches/mustafa_branch/djagen/collector/planethandler.py
Executable file
|
@ -0,0 +1,101 @@
|
||||||
|
#!/usr/bin/python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
from djagen.collector.models import *
|
||||||
|
|
||||||
|
class PlanetHandler():
|
||||||
|
|
||||||
|
def __init__(self,planet):
|
||||||
|
self.channels = planet.channels()
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
for channel in self.channels:
|
||||||
|
self.author_ch = self.__DeliverChannelToDB(channel)
|
||||||
|
for item in channel.items():
|
||||||
|
self.__DeliverItemToDB(item,self.author_ch)
|
||||||
|
self.__DeliverRunTimeToDB()
|
||||||
|
|
||||||
|
def __DeliverChannelToDB(self,channel):
|
||||||
|
try:
|
||||||
|
author = Authors.objects.get(author_id = channel.id )
|
||||||
|
except:
|
||||||
|
author = Authors(author_id=channel.id)
|
||||||
|
finally:
|
||||||
|
author.author_name = channel.name
|
||||||
|
try:
|
||||||
|
author.author_surname = channel.surname
|
||||||
|
except:
|
||||||
|
author.author_surname = ''
|
||||||
|
author.channel_url = channel.url
|
||||||
|
|
||||||
|
try:
|
||||||
|
author.author_face = channel.face
|
||||||
|
except:
|
||||||
|
author.author_face = None
|
||||||
|
try:
|
||||||
|
author.channel_subtitle = channel.subtitle
|
||||||
|
except:
|
||||||
|
author.channel_subtitle = None
|
||||||
|
try:
|
||||||
|
author.channel_title = channel.title
|
||||||
|
except:
|
||||||
|
author.channel_title = None
|
||||||
|
try:
|
||||||
|
author.channel_link = channel.link
|
||||||
|
except:
|
||||||
|
author.channel_link = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
author.channel_url_status = channel.url_status
|
||||||
|
except:
|
||||||
|
author.channel_url_status = None
|
||||||
|
|
||||||
|
if channel.label == "Personal":
|
||||||
|
author.label_personal = 1
|
||||||
|
if channel.label == "LKD":
|
||||||
|
author.label_lkd = 1
|
||||||
|
if channel.label == "Community":
|
||||||
|
author.label_community = 1
|
||||||
|
if channel.label == "Eng":
|
||||||
|
author.label_eng = 1
|
||||||
|
author.save()
|
||||||
|
return author
|
||||||
|
|
||||||
|
def __DeliverItemToDB(self,item,author):
|
||||||
|
try:
|
||||||
|
entry = author.entries_set.get(id_hash = item.id_hash)
|
||||||
|
except:
|
||||||
|
d = item.date
|
||||||
|
entry = author.entries_set.create(
|
||||||
|
id_hash = item.id_hash,
|
||||||
|
date =datetime.datetime(d[0], d[1], d[2], d[3], d[4], d[5]),
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
entry.title = item.title
|
||||||
|
except:
|
||||||
|
entry.title = None
|
||||||
|
try:
|
||||||
|
entry.content_html = item.content
|
||||||
|
except:
|
||||||
|
entry.content_html = None
|
||||||
|
try:
|
||||||
|
entry.content_text = entry.sanitize(item.content)
|
||||||
|
except:
|
||||||
|
entry.content_text = None
|
||||||
|
try:
|
||||||
|
entry.summary = item.summary
|
||||||
|
except:
|
||||||
|
entry.summary = None
|
||||||
|
try:
|
||||||
|
entry.link = item.link
|
||||||
|
except:
|
||||||
|
entry.link = None
|
||||||
|
d = item.date
|
||||||
|
entry.date = datetime.datetime(d[0], d[1], d[2], d[3], d[4], d[5])
|
||||||
|
entry.save()
|
||||||
|
return entry
|
||||||
|
|
||||||
|
def __DeliverRunTimeToDB(self):
|
||||||
|
r = RunTime()
|
||||||
|
r.save()
|
|
@ -49,6 +49,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),'../..'))
|
||||||
|
|
||||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'djagen.settings'
|
os.environ['DJANGO_SETTINGS_MODULE'] = 'djagen.settings'
|
||||||
from djagen.collector.models import *
|
from djagen.collector.models import *
|
||||||
|
from djagen.collector.planethandler import PlanetHandler
|
||||||
|
|
||||||
def config_get(config, section, option, default=None, raw=0, vars=None):
|
def config_get(config, section, option, default=None, raw=0, vars=None):
|
||||||
"""Get a value from the configuration, with a default."""
|
"""Get a value from the configuration, with a default."""
|
||||||
|
@ -167,172 +168,12 @@ def main():
|
||||||
# run the planet
|
# run the planet
|
||||||
my_planet = planet.Planet(config)
|
my_planet = planet.Planet(config)
|
||||||
my_planet.run(planet_name, planet_link, template_files, offline)
|
my_planet.run(planet_name, planet_link, template_files, offline)
|
||||||
|
handler_instance = PlanetHandler(my_planet)
|
||||||
|
handler_instance.run()
|
||||||
|
|
||||||
|
|
||||||
|
#my_planet.generate_all_files(template_files, planet_name,
|
||||||
|
#planet_link, planet_feed, owner_name, owner_email)
|
||||||
## This is where archiving is done! ##
|
|
||||||
#add the current channels to the db
|
|
||||||
channels = my_planet.channels()
|
|
||||||
for channel in channels:
|
|
||||||
### This part seperates surname from name do not activate it if needn't.
|
|
||||||
words = channel.name.split()
|
|
||||||
if len(words) == 1:
|
|
||||||
author_name = words[0]
|
|
||||||
author_surname == None
|
|
||||||
else:
|
|
||||||
author_surname = words[-1]
|
|
||||||
words.pop()
|
|
||||||
tmp_first_name = ''
|
|
||||||
for word in words: tmp_first_name += ' ' + word
|
|
||||||
author_name = tmp_first_name[1:]
|
|
||||||
### ###
|
|
||||||
#print channel.name
|
|
||||||
#author_name = channel.name
|
|
||||||
#author_surname = channel.surname
|
|
||||||
|
|
||||||
try:
|
|
||||||
author_face = channel.face
|
|
||||||
except:
|
|
||||||
author_face = None
|
|
||||||
try:
|
|
||||||
channel_subtitle = channel.subtitle
|
|
||||||
except:
|
|
||||||
channel_subtitle = None
|
|
||||||
try:
|
|
||||||
channel_title = channel.title
|
|
||||||
except:
|
|
||||||
channel_title = None
|
|
||||||
|
|
||||||
channel_url = channel.url
|
|
||||||
|
|
||||||
try:
|
|
||||||
channel_link = channel.link
|
|
||||||
except:
|
|
||||||
channel_link = None
|
|
||||||
|
|
||||||
try:
|
|
||||||
channel_urlstatus = channel.url_status
|
|
||||||
except:
|
|
||||||
channel_urlstatus = None
|
|
||||||
label = channel.label
|
|
||||||
|
|
||||||
label_personal = 0
|
|
||||||
label_lkd = 0
|
|
||||||
label_community = 0
|
|
||||||
label_eng = 0
|
|
||||||
if label == "Personal":
|
|
||||||
label_personal = 1
|
|
||||||
if label == "LKD":
|
|
||||||
label_lkd = 1
|
|
||||||
if label == "Community":
|
|
||||||
label_community = 1
|
|
||||||
if label == "Eng":
|
|
||||||
label_eng = 1
|
|
||||||
id = channel.id
|
|
||||||
#print ("id is ", id)
|
|
||||||
|
|
||||||
try:
|
|
||||||
#update the values with the ones at the config file
|
|
||||||
author = Authors.objects.get(author_id=id)
|
|
||||||
|
|
||||||
author.author_name = author_name
|
|
||||||
author.author_surname = author_surname
|
|
||||||
author.channel_url = channel_url
|
|
||||||
|
|
||||||
try:
|
|
||||||
author.author_face = author_face
|
|
||||||
except:
|
|
||||||
author.author_face = None
|
|
||||||
try:
|
|
||||||
author.channel_subtitle = channel_subtitle
|
|
||||||
except:
|
|
||||||
author.channel_subtitle = None
|
|
||||||
try:
|
|
||||||
author.channel_title = channel_title
|
|
||||||
except:
|
|
||||||
author.channel_title = None
|
|
||||||
try:
|
|
||||||
author.channel_link = channel_link
|
|
||||||
except:
|
|
||||||
author.channel_link = None
|
|
||||||
try:
|
|
||||||
author.channel_url_status = channel_urlstatus
|
|
||||||
except:
|
|
||||||
author.channel_url_status = None
|
|
||||||
|
|
||||||
author.label_personal = label_personal
|
|
||||||
author.label_lkd = label_lkd
|
|
||||||
author.label_community = label_community
|
|
||||||
author.label_eng = label_eng
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
except Exception, ex:
|
|
||||||
#print ex
|
|
||||||
author = Authors(author_id=id, author_name=author_name, author_surname=author_surname, author_face=author_face, channel_subtitle=channel_subtitle, channel_title=channel_title, channel_url=channel_url, channel_link=channel_link, channel_urlstatus=channel_urlstatus, label_personal=label_personal, label_lkd=label_lkd, label_community=label_community, label_eng=label_eng)
|
|
||||||
|
|
||||||
|
|
||||||
author.save()
|
|
||||||
|
|
||||||
#entry issues
|
|
||||||
items = channel.items()
|
|
||||||
for item in items:
|
|
||||||
id_hash = item.id_hash
|
|
||||||
|
|
||||||
try:
|
|
||||||
#Only check for get if that fails then switch to create new entry mode.
|
|
||||||
entry = author.entries_set.get(id_hash = item.id_hash)
|
|
||||||
|
|
||||||
try:
|
|
||||||
entry.title = item.title
|
|
||||||
except:
|
|
||||||
#print "title"
|
|
||||||
entry.title = None
|
|
||||||
try:
|
|
||||||
entry.content_html = item.content
|
|
||||||
except:
|
|
||||||
entry.content_html = None
|
|
||||||
try:
|
|
||||||
entry.content_text = entry.sanitize(item.content)
|
|
||||||
except:
|
|
||||||
entry.content_text = None
|
|
||||||
try:
|
|
||||||
|
|
||||||
entry.summary = item.summary
|
|
||||||
except:
|
|
||||||
#print "summary"
|
|
||||||
entry.summary = None
|
|
||||||
try:
|
|
||||||
|
|
||||||
entry.link = item.link
|
|
||||||
except:
|
|
||||||
#print "link"
|
|
||||||
entry.link = None
|
|
||||||
|
|
||||||
d = item.date
|
|
||||||
|
|
||||||
|
|
||||||
entry.date = datetime.datetime(d[0], d[1], d[2], d[3], d[4], d[5])
|
|
||||||
|
|
||||||
|
|
||||||
except:
|
|
||||||
content_html = item.content
|
|
||||||
#content_text = entry.sanitize(content_html)
|
|
||||||
d = item.date
|
|
||||||
if not item.has_key('summary'): summary = None
|
|
||||||
else: summary = item.summary
|
|
||||||
entry = author.entries_set.create(id_hash= item.id_hash, title=item.title, content_html=item.content, summary=summary, link=item.link, date=datetime.datetime(d[0], d[1], d[2], d[3], d[4], d[5]))
|
|
||||||
entry.content_text = entry.sanitize(content_html)
|
|
||||||
|
|
||||||
entry.save()
|
|
||||||
|
|
||||||
#datetime issue
|
|
||||||
r = RunTime()
|
|
||||||
r.save()
|
|
||||||
|
|
||||||
my_planet.generate_all_files(template_files, planet_name,
|
|
||||||
planet_link, planet_feed, owner_name, owner_email)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
Loading…
Reference in New Issue
Block a user