+
diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/feeds.html.tmplc b/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/feeds.html.tmplc
new file mode 100755
index 0000000..155f4e4
Binary files /dev/null and b/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/feeds.html.tmplc differ
diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/foafroll.xml.tmpl b/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/foafroll.xml.tmpl
new file mode 100755
index 0000000..f344738
--- /dev/null
+++ b/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/foafroll.xml.tmpl
@@ -0,0 +1,31 @@
+
+-
+
+
+ 
" title="" />
+
+ 
+
+
+
+ " title="subscribe">
+
+ 
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+ 
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/index.html.tmplc b/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/index.html.tmplc
new file mode 100755
index 0000000..259931d
Binary files /dev/null and b/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/index.html.tmplc differ
diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/opml.xml.tmpl b/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/opml.xml.tmpl
new file mode 100755
index 0000000..50bbabe
--- /dev/null
+++ b/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/opml.xml.tmpl
@@ -0,0 +1,16 @@
+
+
+
+
+ Gezegen Hakkında
++ Linux Gezegeni, Türkiye'de Linux ve Özgür Yazılım konusunda çalışmalar yapan arkadaşlarımızın internet üzerindeki günlüklerini bir tek sayfadan okumamızı ve kendi dünyalarına ulaşmamızı sağlayan basit bir web sitesidir. +
++ Gezegen kurallarına ulaşmak için tıklayın.. +
++ Gezegeni Planet ile oluşturuyoruz, tasarım Steven Garrity'nin eseri. +
+
+
+
+
+
+
+![]()
+
+
+
+
+
+
+
+
+
+
+![]()
+
+
+
+
+
+
+
+
+
+
+ Gezegen Sakinleri
+ +
+
+
+ Diğer Gezegenler
+ +
+
+
+ Güncelleme
+Gezegen her 10 dakikada bir yenilenir.
+
+ Son güncelleme:
+
+
+
+ İletişim
++ Linux Gezegeni Gezegen Ekibi tarafından yönetilmektedir, Gezegen hakkındaki sorularınızı ve Gezegen'e iniş başvurularınızı e-posta ile iletebilirsiniz. +
++ Gezegene iniş başvurularınızda Gezegen Kuralları'na uyan RSS/Atom beslemenizi ve gezegen içerisinde kullanmak istediğiniz (en fazla 80x80 çözünürlüklü) resminizi (bir başka deyişle hackergotchi); varsa jabber adresinizle birlikte e-posta yoluyla göndermenizi rica ediyoruz. +
+-
+
-
+">
+
+ +" height="24" alt=""> +
+ 
++
+
+
+
+
+
+
+
+
+
+
+ ">
+
+
+ ">
+
+
+
+
+
+
+
+Copyright © 2003-2007, The GNOME Project
+Blog entries aggregated on this page are owned by, and represent the opinion of the author.
+Optimised for standards. Hosted by Red Hat.
+
+
+
+
+
+
diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/simple.html.tmplc b/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/simple.html.tmplc
new file mode 100755
index 0000000..d466e42
Binary files /dev/null and b/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/simple.html.tmplc differ
diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/zaman.sh b/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/zaman.sh
new file mode 100755
index 0000000..e0c9a2b
--- /dev/null
+++ b/DJAGEN/branches/mustafa_branch/djagen/gezegen/gezegen/zaman.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+while read x
+do
+ echo "$(date)::$x"
+done
diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet-cache.py b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet-cache.py
new file mode 100755
index 0000000..9334583
--- /dev/null
+++ b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet-cache.py
@@ -0,0 +1,194 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""Planet cache tool.
+
+"""
+
+__authors__ = [ "Scott James Remnant +Blog entries aggregated on this page are owned by, and represent the opinion of the author.
+Optimised for standards. Hosted by Red Hat.
+
%s
'
+ data = minidom.parseString((div % text.encode('utf-8')))
+
+ if text.find('<') < 0:
+ # plain text
+ node.removeAttribute('type')
+ text = data.documentElement.childNodes[0].nodeValue
+ node.childNodes[0].replaceWholeText(text)
+
+ elif len(text) > 80:
+ # xhtml
+ node.setAttribute('type', 'xhtml')
+ node.removeChild(node.childNodes[0])
+ node.appendChild(data.documentElement)
+
+ except ExpatError:
+ # leave as html
+ pass
+
+ else:
+ # recurse
+ retype(node)
+
+ if parent.nodeName == 'entry':
+ relativize(parent)
+
+if __name__ == '__main__':
+
+ # run styler on each file mention on the command line
+ import sys
+ for feed in sys.argv[1:]:
+ doc = minidom.parse(feed)
+ doc.normalize()
+ retype(doc.documentElement)
+ open(feed,'w').write(doc.toxml('utf-8'))
diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/cache.py b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/cache.py
new file mode 100755
index 0000000..dfc529b
--- /dev/null
+++ b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/cache.py
@@ -0,0 +1,306 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""Item cache.
+
+Between runs of Planet we need somewhere to store the feed information
+we parsed, this is so we don't lose information when a particular feed
+goes away or is too short to hold enough items.
+
+This module provides the code to handle this cache transparently enough
+that the rest of the code can take the persistance for granted.
+"""
+
+import os
+import re
+
+
+# Regular expressions to sanitise cache filenames
+re_url_scheme = re.compile(r'^[^:]*://')
+re_slash = re.compile(r'[?/]+')
+re_initial_cruft = re.compile(r'^[,.]*')
+re_final_cruft = re.compile(r'[,.]*$')
+
+
+class CachedInfo:
+ """Cached information.
+
+ This class is designed to hold information that is stored in a cache
+ between instances. It can act both as a dictionary (c['foo']) and
+ as an object (c.foo) to get and set values and supports both string
+ and date values.
+
+ If you wish to support special fields you can derive a class off this
+ and implement get_FIELD and set_FIELD functions which will be
+ automatically called.
+ """
+ STRING = "string"
+ DATE = "date"
+ NULL = "null"
+
+ def __init__(self, cache, id_, root=0):
+ self._type = {}
+ self._value = {}
+ self._cached = {}
+
+ self._cache = cache
+ self._id = id_.replace(" ", "%20")
+ self._root = root
+
+ def cache_key(self, key):
+ """Return the cache key name for the given key."""
+ key = key.replace(" ", "_")
+ if self._root:
+ return key
+ else:
+ return self._id + " " + key
+
+ def cache_read(self):
+ """Read information from the cache."""
+ if self._root:
+ keys_key = " keys"
+ else:
+ keys_key = self._id
+
+ if self._cache.has_key(keys_key):
+ keys = self._cache[keys_key].split(" ")
+ else:
+ return
+
+ for key in keys:
+ cache_key = self.cache_key(key)
+ if not self._cached.has_key(key) or self._cached[key]:
+ # Key either hasn't been loaded, or is one for the cache
+ self._value[key] = self._cache[cache_key]
+ self._type[key] = self._cache[cache_key + " type"]
+ self._cached[key] = 1
+
+ def cache_write(self, sync=1):
+ """Write information to the cache."""
+ self.cache_clear(sync=0)
+
+ keys = []
+ for key in self.keys():
+ cache_key = self.cache_key(key)
+ if not self._cached[key]:
+ if self._cache.has_key(cache_key):
+ # Non-cached keys need to be cleared
+ del(self._cache[cache_key])
+ del(self._cache[cache_key + " type"])
+ continue
+
+ keys.append(key)
+ self._cache[cache_key] = self._value[key]
+ self._cache[cache_key + " type"] = self._type[key]
+
+ if self._root:
+ keys_key = " keys"
+ else:
+ keys_key = self._id
+
+ self._cache[keys_key] = " ".join(keys)
+ if sync:
+ self._cache.sync()
+
+ def cache_clear(self, sync=1):
+ """Remove information from the cache."""
+ if self._root:
+ keys_key = " keys"
+ else:
+ keys_key = self._id
+
+ if self._cache.has_key(keys_key):
+ keys = self._cache[keys_key].split(" ")
+ del(self._cache[keys_key])
+ else:
+ return
+
+ for key in keys:
+ cache_key = self.cache_key(key)
+ del(self._cache[cache_key])
+ del(self._cache[cache_key + " type"])
+
+ if sync:
+ self._cache.sync()
+
+ def has_key(self, key):
+ """Check whether the key exists."""
+ key = key.replace(" ", "_")
+ return self._value.has_key(key)
+
+ def key_type(self, key):
+ """Return the key type."""
+ key = key.replace(" ", "_")
+ return self._type[key]
+
+ def set(self, key, value, cached=1):
+ """Set the value of the given key.
+
+ If a set_KEY function exists that is called otherwise the
+ string function is called and the date function if that fails
+ (it nearly always will).
+ """
+ key = key.replace(" ", "_")
+
+ try:
+ func = getattr(self, "set_" + key)
+ except AttributeError:
+ pass
+ else:
+ return func(key, value)
+
+ if value == None:
+ return self.set_as_null(key, value)
+ else:
+ try:
+ return self.set_as_string(key, value)
+ except TypeError:
+ return self.set_as_date(key, value)
+
+ def get(self, key):
+ """Return the value of the given key.
+
+ If a get_KEY function exists that is called otherwise the
+ correctly typed function is called if that exists.
+ """
+ key = key.replace(" ", "_")
+
+ try:
+ func = getattr(self, "get_" + key)
+ except AttributeError:
+ pass
+ else:
+ return func(key)
+
+ try:
+ func = getattr(self, "get_as_" + self._type[key])
+ except AttributeError:
+ pass
+ else:
+ return func(key)
+
+ return self._value[key]
+
+ def set_as_string(self, key, value, cached=1):
+ """Set the key to the string value.
+
+ The value is converted to UTF-8 if it is a Unicode string, otherwise
+ it's assumed to have failed decoding (feedparser tries pretty hard)
+ so has all non-ASCII characters stripped.
+ """
+ value = utf8(value)
+
+ key = key.replace(" ", "_")
+ self._value[key] = value
+ self._type[key] = self.STRING
+ self._cached[key] = cached
+
+ def get_as_string(self, key):
+ """Return the key as a string value."""
+ key = key.replace(" ", "_")
+ if not self.has_key(key):
+ raise KeyError, key
+
+ return self._value[key]
+
+ def set_as_date(self, key, value, cached=1):
+ """Set the key to the date value.
+
+ The date should be a 9-item tuple as returned by time.gmtime().
+ """
+ value = " ".join([ str(s) for s in value ])
+
+ key = key.replace(" ", "_")
+ self._value[key] = value
+ self._type[key] = self.DATE
+ self._cached[key] = cached
+
+ def get_as_date(self, key):
+ """Return the key as a date value."""
+ key = key.replace(" ", "_")
+ if not self.has_key(key):
+ raise KeyError, key
+
+ value = self._value[key]
+ return tuple([ int(i) for i in value.split(" ") ])
+
+ def set_as_null(self, key, value, cached=1):
+ """Set the key to the null value.
+
+ This only exists to make things less magic.
+ """
+ key = key.replace(" ", "_")
+ self._value[key] = ""
+ self._type[key] = self.NULL
+ self._cached[key] = cached
+
+ def get_as_null(self, key):
+ """Return the key as the null value."""
+ key = key.replace(" ", "_")
+ if not self.has_key(key):
+ raise KeyError, key
+
+ return None
+
+ def del_key(self, key):
+ """Delete the given key."""
+ key = key.replace(" ", "_")
+ if not self.has_key(key):
+ raise KeyError, key
+
+ del(self._value[key])
+ del(self._type[key])
+ del(self._cached[key])
+
+ def keys(self):
+ """Return the list of cached keys."""
+ return self._value.keys()
+
+ def __iter__(self):
+ """Iterate the cached keys."""
+ return iter(self._value.keys())
+
+ # Special methods
+ __contains__ = has_key
+ __setitem__ = set_as_string
+ __getitem__ = get
+ __delitem__ = del_key
+ __delattr__ = del_key
+
+ def __setattr__(self, key, value):
+ if key.startswith("_"):
+ self.__dict__[key] = value
+ else:
+ self.set(key, value)
+
+ def __getattr__(self, key):
+ if self.has_key(key):
+ return self.get(key)
+ else:
+ raise AttributeError, key
+
+
+def filename(directory, filename):
+ """Return a filename suitable for the cache.
+
+ Strips dangerous and common characters to create a filename we
+ can use to store the cache in.
+ """
+ filename = re_url_scheme.sub("", filename)
+ filename = re_slash.sub(",", filename)
+ filename = re_initial_cruft.sub("", filename)
+ filename = re_final_cruft.sub("", filename)
+
+ return os.path.join(directory, filename)
+
+def utf8(value):
+ """Return the value as a UTF-8 string."""
+ if type(value) == type(u''):
+ return value.encode("utf-8")
+ else:
+ try:
+ return unicode(value, "utf-8").encode("utf-8")
+ except UnicodeError:
+ try:
+ return unicode(value, "iso-8859-1").encode("utf-8")
+ except UnicodeError:
+ return unicode(value, "ascii", "replace").encode("utf-8")
diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/compat_logging/__init__.py b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/compat_logging/__init__.py
new file mode 100755
index 0000000..3bd0c6d
--- /dev/null
+++ b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/compat_logging/__init__.py
@@ -0,0 +1,1196 @@
+# Copyright 2001-2002 by Vinay Sajip. All Rights Reserved.
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose and without fee is hereby granted,
+# provided that the above copyright notice appear in all copies and that
+# both that copyright notice and this permission notice appear in
+# supporting documentation, and that the name of Vinay Sajip
+# not be used in advertising or publicity pertaining to distribution
+# of the software without specific, written prior permission.
+# VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
+# ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR
+# ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
+# IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Logging package for Python. Based on PEP 282 and comments thereto in
+comp.lang.python, and influenced by Apache's log4j system.
+
+Should work under Python versions >= 1.5.2, except that source line
+information is not available unless 'sys._getframe()' is.
+
+Copyright (C) 2001-2002 Vinay Sajip. All Rights Reserved.
+
+To use, simply 'import logging' and log away!
+"""
+
+import sys, os, types, time, string, cStringIO
+
+try:
+ import thread
+ import threading
+except ImportError:
+ thread = None
+
+__author__ = "Vinay Sajip and
+ # only if all the remaining content is nested underneath it.
+ # This means that the divs would be retained in the following:
+ #
foo
bar
+ if pieces and (pieces[0] == '' or pieces[0].startswith('
':
+ depth = 0
+ for piece in pieces[:-1]:
+ if piece.startswith(''):
+ depth += 1
+ else:
+ pieces = pieces[1:-1]
+
+ output = ''.join(pieces)
+ if stripWhitespace:
+ output = output.strip()
+ if not expectingText: return output
+
+ # decode base64 content
+ if base64 and self.contentparams.get('base64', 0):
+ try:
+ output = base64.decodestring(output)
+ except binascii.Error:
+ pass
+ except binascii.Incomplete:
+ pass
+
+ # resolve relative URIs
+ if (element in self.can_be_relative_uri) and output:
+ output = self.resolveURI(output)
+
+ # decode entities within embedded markup
+ if not self.contentparams.get('base64', 0):
+ output = self.decodeEntities(element, output)
+
+ # remove temporary cruft from contentparams
+ try:
+ del self.contentparams['mode']
+ except KeyError:
+ pass
+ try:
+ del self.contentparams['base64']
+ except KeyError:
+ pass
+
+ # resolve relative URIs within embedded markup
+ if self.mapContentType(self.contentparams.get('type', 'text/html')) in self.html_types:
+ if element in self.can_contain_relative_uris:
+ output = _resolveRelativeURIs(output, self.baseuri, self.encoding)
+
+ # sanitize embedded markup
+ if self.mapContentType(self.contentparams.get('type', 'text/html')) in self.html_types:
+ if element in self.can_contain_dangerous_markup:
+ output = _sanitizeHTML(output, self.encoding)
+
+ if self.encoding and type(output) != type(u''):
+ try:
+ output = unicode(output, self.encoding)
+ except:
+ pass
+
+ # address common error where people take data that is already
+ # utf-8, presume that it is iso-8859-1, and re-encode it.
+ if self.encoding=='utf-8' and type(output) == type(u''):
+ try:
+ output = unicode(output.encode('iso-8859-1'), 'utf-8')
+ except:
+ pass
+
+ # map win-1252 extensions to the proper code points
+ if type(output) == type(u''):
+ output = u''.join([c in cp1252 and cp1252[c] or c for c in output])
+
+ # categories/tags/keywords/whatever are handled in _end_category
+ if element == 'category':
+ return output
+
+ # store output in appropriate place(s)
+ if self.inentry and not self.insource:
+ if element == 'content':
+ self.entries[-1].setdefault(element, [])
+ contentparams = copy.deepcopy(self.contentparams)
+ contentparams['value'] = output
+ self.entries[-1][element].append(contentparams)
+ elif element == 'link':
+ self.entries[-1][element] = output
+ if output:
+ self.entries[-1]['links'][-1]['href'] = output
+ else:
+ if element == 'description':
+ element = 'summary'
+ self.entries[-1][element] = output
+ if self.incontent:
+ contentparams = copy.deepcopy(self.contentparams)
+ contentparams['value'] = output
+ self.entries[-1][element + '_detail'] = contentparams
+ elif (self.infeed or self.insource) and (not self.intextinput) and (not self.inimage):
+ context = self._getContext()
+ if element == 'description':
+ element = 'subtitle'
+ context[element] = output
+ if element == 'link':
+ context['links'][-1]['href'] = output
+ elif self.incontent:
+ contentparams = copy.deepcopy(self.contentparams)
+ contentparams['value'] = output
+ context[element + '_detail'] = contentparams
+ return output
+
+ def pushContent(self, tag, attrsD, defaultContentType, expectingText):
+ self.incontent += 1
+ self.contentparams = FeedParserDict({
+ 'type': self.mapContentType(attrsD.get('type', defaultContentType)),
+ 'language': self.lang,
+ 'base': self.baseuri})
+ self.contentparams['base64'] = self._isBase64(attrsD, self.contentparams)
+ self.push(tag, expectingText)
+
+ def popContent(self, tag):
+ value = self.pop(tag)
+ self.incontent -= 1
+ self.contentparams.clear()
+ return value
+
+ def _mapToStandardPrefix(self, name):
+ colonpos = name.find(':')
+ if colonpos <> -1:
+ prefix = name[:colonpos]
+ suffix = name[colonpos+1:]
+ prefix = self.namespacemap.get(prefix, prefix)
+ name = prefix + ':' + suffix
+ return name
+
+ def _getAttribute(self, attrsD, name):
+ return attrsD.get(self._mapToStandardPrefix(name))
+
+ def _isBase64(self, attrsD, contentparams):
+ if attrsD.get('mode', '') == 'base64':
+ return 1
+ if self.contentparams['type'].startswith('text/'):
+ return 0
+ if self.contentparams['type'].endswith('+xml'):
+ return 0
+ if self.contentparams['type'].endswith('/xml'):
+ return 0
+ return 1
+
+ def _itsAnHrefDamnIt(self, attrsD):
+ href = attrsD.get('url', attrsD.get('uri', attrsD.get('href', None)))
+ if href:
+ try:
+ del attrsD['url']
+ except KeyError:
+ pass
+ try:
+ del attrsD['uri']
+ except KeyError:
+ pass
+ attrsD['href'] = href
+ return attrsD
+
+ def _save(self, key, value):
+ context = self._getContext()
+ context.setdefault(key, value)
+
+ def _start_rss(self, attrsD):
+ versionmap = {'0.91': 'rss091u',
+ '0.92': 'rss092',
+ '0.93': 'rss093',
+ '0.94': 'rss094'}
+ if not self.version:
+ attr_version = attrsD.get('version', '')
+ version = versionmap.get(attr_version)
+ if version:
+ self.version = version
+ elif attr_version.startswith('2.'):
+ self.version = 'rss20'
+ else:
+ self.version = 'rss'
+
+ def _start_dlhottitles(self, attrsD):
+ self.version = 'hotrss'
+
+ def _start_channel(self, attrsD):
+ self.infeed = 1
+ self._cdf_common(attrsD)
+ _start_feedinfo = _start_channel
+
+ def _cdf_common(self, attrsD):
+ if attrsD.has_key('lastmod'):
+ self._start_modified({})
+ self.elementstack[-1][-1] = attrsD['lastmod']
+ self._end_modified()
+ if attrsD.has_key('href'):
+ self._start_link({})
+ self.elementstack[-1][-1] = attrsD['href']
+ self._end_link()
+
+ def _start_feed(self, attrsD):
+ self.infeed = 1
+ versionmap = {'0.1': 'atom01',
+ '0.2': 'atom02',
+ '0.3': 'atom03'}
+ if not self.version:
+ attr_version = attrsD.get('version')
+ version = versionmap.get(attr_version)
+ if version:
+ self.version = version
+ else:
+ self.version = 'atom'
+
+ def _end_channel(self):
+ self.infeed = 0
+ _end_feed = _end_channel
+
+ def _start_image(self, attrsD):
+ self.inimage = 1
+ self.push('image', 0)
+ context = self._getContext()
+ context.setdefault('image', FeedParserDict())
+
+ def _end_image(self):
+ self.pop('image')
+ self.inimage = 0
+
+ def _start_textinput(self, attrsD):
+ self.intextinput = 1
+ self.push('textinput', 0)
+ context = self._getContext()
+ context.setdefault('textinput', FeedParserDict())
+ _start_textInput = _start_textinput
+
+ def _end_textinput(self):
+ self.pop('textinput')
+ self.intextinput = 0
+ _end_textInput = _end_textinput
+
+ def _start_author(self, attrsD):
+ self.inauthor = 1
+ self.push('author', 1)
+ _start_managingeditor = _start_author
+ _start_dc_author = _start_author
+ _start_dc_creator = _start_author
+ _start_itunes_author = _start_author
+
+ def _end_author(self):
+ self.pop('author')
+ self.inauthor = 0
+ self._sync_author_detail()
+ _end_managingeditor = _end_author
+ _end_dc_author = _end_author
+ _end_dc_creator = _end_author
+ _end_itunes_author = _end_author
+
+ def _start_itunes_owner(self, attrsD):
+ self.inpublisher = 1
+ self.push('publisher', 0)
+
+ def _end_itunes_owner(self):
+ self.pop('publisher')
+ self.inpublisher = 0
+ self._sync_author_detail('publisher')
+
+ def _start_contributor(self, attrsD):
+ self.incontributor = 1
+ context = self._getContext()
+ context.setdefault('contributors', [])
+ context['contributors'].append(FeedParserDict())
+ self.push('contributor', 0)
+
+ def _end_contributor(self):
+ self.pop('contributor')
+ self.incontributor = 0
+
+ def _start_dc_contributor(self, attrsD):
+ self.incontributor = 1
+ context = self._getContext()
+ context.setdefault('contributors', [])
+ context['contributors'].append(FeedParserDict())
+ self.push('name', 0)
+
+ def _end_dc_contributor(self):
+ self._end_name()
+ self.incontributor = 0
+
+ def _start_name(self, attrsD):
+ self.push('name', 0)
+ _start_itunes_name = _start_name
+
+ def _end_name(self):
+ value = self.pop('name')
+ if self.inpublisher:
+ self._save_author('name', value, 'publisher')
+ elif self.inauthor:
+ self._save_author('name', value)
+ elif self.incontributor:
+ self._save_contributor('name', value)
+ elif self.intextinput:
+ context = self._getContext()
+ context['textinput']['name'] = value
+ _end_itunes_name = _end_name
+
+ def _start_width(self, attrsD):
+ self.push('width', 0)
+
+ def _end_width(self):
+ value = self.pop('width')
+ try:
+ value = int(value)
+ except:
+ value = 0
+ if self.inimage:
+ context = self._getContext()
+ context['image']['width'] = value
+
+ def _start_height(self, attrsD):
+ self.push('height', 0)
+
+ def _end_height(self):
+ value = self.pop('height')
+ try:
+ value = int(value)
+ except:
+ value = 0
+ if self.inimage:
+ context = self._getContext()
+ context['image']['height'] = value
+
+ def _start_url(self, attrsD):
+ self.push('href', 1)
+ _start_homepage = _start_url
+ _start_uri = _start_url
+
+ def _end_url(self):
+ value = self.pop('href')
+ if self.inauthor:
+ self._save_author('href', value)
+ elif self.incontributor:
+ self._save_contributor('href', value)
+ elif self.inimage:
+ context = self._getContext()
+ context['image']['href'] = value
+ elif self.intextinput:
+ context = self._getContext()
+ context['textinput']['link'] = value
+ _end_homepage = _end_url
+ _end_uri = _end_url
+
+ def _start_email(self, attrsD):
+ self.push('email', 0)
+ _start_itunes_email = _start_email
+
+ def _end_email(self):
+ value = self.pop('email')
+ if self.inpublisher:
+ self._save_author('email', value, 'publisher')
+ elif self.inauthor:
+ self._save_author('email', value)
+ elif self.incontributor:
+ self._save_contributor('email', value)
+ _end_itunes_email = _end_email
+
+ def _getContext(self):
+ if self.insource:
+ context = self.sourcedata
+ elif self.inentry:
+ context = self.entries[-1]
+ else:
+ context = self.feeddata
+ return context
+
+ def _save_author(self, key, value, prefix='author'):
+ context = self._getContext()
+ context.setdefault(prefix + '_detail', FeedParserDict())
+ context[prefix + '_detail'][key] = value
+ self._sync_author_detail()
+
+ def _save_contributor(self, key, value):
+ context = self._getContext()
+ context.setdefault('contributors', [FeedParserDict()])
+ context['contributors'][-1][key] = value
+
+ def _sync_author_detail(self, key='author'):
+ context = self._getContext()
+ detail = context.get('%s_detail' % key)
+ if detail:
+ name = detail.get('name')
+ email = detail.get('email')
+ if name and email:
+ context[key] = '%s (%s)' % (name, email)
+ elif name:
+ context[key] = name
+ elif email:
+ context[key] = email
+ else:
+ author = context.get(key)
+ if not author: return
+ emailmatch = re.search(r'''(([a-zA-Z0-9\_\-\.\+]+)@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.)|(([a-zA-Z0-9\-]+\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\]?))''', author)
+ if not emailmatch: return
+ email = emailmatch.group(0)
+ # probably a better way to do the following, but it passes all the tests
+ author = author.replace(email, '')
+ author = author.replace('()', '')
+ author = author.strip()
+ if author and (author[0] == '('):
+ author = author[1:]
+ if author and (author[-1] == ')'):
+ author = author[:-1]
+ author = author.strip()
+ context.setdefault('%s_detail' % key, FeedParserDict())
+ context['%s_detail' % key]['name'] = author
+ context['%s_detail' % key]['email'] = email
+
+ def _start_subtitle(self, attrsD):
+ self.pushContent('subtitle', attrsD, 'text/plain', 1)
+ _start_tagline = _start_subtitle
+ _start_itunes_subtitle = _start_subtitle
+
+ def _end_subtitle(self):
+ self.popContent('subtitle')
+ _end_tagline = _end_subtitle
+ _end_itunes_subtitle = _end_subtitle
+
+ def _start_rights(self, attrsD):
+ self.pushContent('rights', attrsD, 'text/plain', 1)
+ _start_dc_rights = _start_rights
+ _start_copyright = _start_rights
+
+ def _end_rights(self):
+ self.popContent('rights')
+ _end_dc_rights = _end_rights
+ _end_copyright = _end_rights
+
+ def _start_item(self, attrsD):
+ self.entries.append(FeedParserDict())
+ self.push('item', 0)
+ self.inentry = 1
+ self.guidislink = 0
+ id = self._getAttribute(attrsD, 'rdf:about')
+ if id:
+ context = self._getContext()
+ context['id'] = id
+ self._cdf_common(attrsD)
+ _start_entry = _start_item
+ _start_product = _start_item
+
+ def _end_item(self):
+ self.pop('item')
+ self.inentry = 0
+ _end_entry = _end_item
+
+ def _start_dc_language(self, attrsD):
+ self.push('language', 1)
+ _start_language = _start_dc_language
+
+ def _end_dc_language(self):
+ self.lang = self.pop('language')
+ _end_language = _end_dc_language
+
+ def _start_dc_publisher(self, attrsD):
+ self.push('publisher', 1)
+ _start_webmaster = _start_dc_publisher
+
+ def _end_dc_publisher(self):
+ self.pop('publisher')
+ self._sync_author_detail('publisher')
+ _end_webmaster = _end_dc_publisher
+
+ def _start_published(self, attrsD):
+ self.push('published', 1)
+ _start_dcterms_issued = _start_published
+ _start_issued = _start_published
+
+ def _end_published(self):
+ value = self.pop('published')
+ self._save('published_parsed', _parse_date(value))
+ _end_dcterms_issued = _end_published
+ _end_issued = _end_published
+
+ def _start_updated(self, attrsD):
+ self.push('updated', 1)
+ _start_modified = _start_updated
+ _start_dcterms_modified = _start_updated
+ _start_pubdate = _start_updated
+ _start_dc_date = _start_updated
+
+ def _end_updated(self):
+ value = self.pop('updated')
+ parsed_value = _parse_date(value)
+ self._save('updated_parsed', parsed_value)
+ _end_modified = _end_updated
+ _end_dcterms_modified = _end_updated
+ _end_pubdate = _end_updated
+ _end_dc_date = _end_updated
+
+ def _start_created(self, attrsD):
+ self.push('created', 1)
+ _start_dcterms_created = _start_created
+
+ def _end_created(self):
+ value = self.pop('created')
+ self._save('created_parsed', _parse_date(value))
+ _end_dcterms_created = _end_created
+
+ def _start_expirationdate(self, attrsD):
+ self.push('expired', 1)
+
+ def _end_expirationdate(self):
+ self._save('expired_parsed', _parse_date(self.pop('expired')))
+
+ def _start_cc_license(self, attrsD):
+ self.push('license', 1)
+ value = self._getAttribute(attrsD, 'rdf:resource')
+ if value:
+ self.elementstack[-1][2].append(value)
+ self.pop('license')
+
+ def _start_creativecommons_license(self, attrsD):
+ self.push('license', 1)
+
+ def _end_creativecommons_license(self):
+ self.pop('license')
+
+ def _addTag(self, term, scheme, label):
+ context = self._getContext()
+ tags = context.setdefault('tags', [])
+ if (not term) and (not scheme) and (not label): return
+ value = FeedParserDict({'term': term, 'scheme': scheme, 'label': label})
+ if value not in tags:
+ tags.append(FeedParserDict({'term': term, 'scheme': scheme, 'label': label}))
+
+ def _start_category(self, attrsD):
+ if _debug: sys.stderr.write('entering _start_category with %s\n' % repr(attrsD))
+ term = attrsD.get('term')
+ scheme = attrsD.get('scheme', attrsD.get('domain'))
+ label = attrsD.get('label')
+ self._addTag(term, scheme, label)
+ self.push('category', 1)
+ _start_dc_subject = _start_category
+ _start_keywords = _start_category
+
+ def _end_itunes_keywords(self):
+ for term in self.pop('itunes_keywords').split():
+ self._addTag(term, 'http://www.itunes.com/', None)
+
+ def _start_itunes_category(self, attrsD):
+ self._addTag(attrsD.get('text'), 'http://www.itunes.com/', None)
+ self.push('category', 1)
+
+ def _end_category(self):
+ value = self.pop('category')
+ if not value: return
+ context = self._getContext()
+ tags = context['tags']
+ if value and len(tags) and not tags[-1]['term']:
+ tags[-1]['term'] = value
+ else:
+ self._addTag(value, None, None)
+ _end_dc_subject = _end_category
+ _end_keywords = _end_category
+ _end_itunes_category = _end_category
+
+ def _start_cloud(self, attrsD):
+ self._getContext()['cloud'] = FeedParserDict(attrsD)
+
+ def _start_link(self, attrsD):
+ attrsD.setdefault('rel', 'alternate')
+ attrsD.setdefault('type', 'text/html')
+ attrsD = self._itsAnHrefDamnIt(attrsD)
+ if attrsD.has_key('href'):
+ attrsD['href'] = self.resolveURI(attrsD['href'])
+ expectingText = self.infeed or self.inentry or self.insource
+ context = self._getContext()
+ context.setdefault('links', [])
+ context['links'].append(FeedParserDict(attrsD))
+ if attrsD['rel'] == 'enclosure':
+ self._start_enclosure(attrsD)
+ if attrsD.has_key('href'):
+ expectingText = 0
+ if (attrsD.get('rel') == 'alternate') and (self.mapContentType(attrsD.get('type')) in self.html_types):
+ context['link'] = attrsD['href']
+ else:
+ self.push('link', expectingText)
+ _start_producturl = _start_link
+
+ def _end_link(self):
+ value = self.pop('link')
+ context = self._getContext()
+ if self.intextinput:
+ context['textinput']['link'] = value
+ if self.inimage:
+ context['image']['link'] = value
+ _end_producturl = _end_link
+
+ def _start_guid(self, attrsD):
+ self.guidislink = (attrsD.get('ispermalink', 'true') == 'true')
+ self.push('id', 1)
+
+ def _end_guid(self):
+ value = self.pop('id')
+ self._save('guidislink', self.guidislink and not self._getContext().has_key('link'))
+ if self.guidislink:
+ # guid acts as link, but only if 'ispermalink' is not present or is 'true',
+ # and only if the item doesn't already have a link element
+ self._save('link', value)
+
+ def _start_title(self, attrsD):
+ self.pushContent('title', attrsD, 'text/plain', self.infeed or self.inentry or self.insource)
+ _start_dc_title = _start_title
+ _start_media_title = _start_title
+
+ def _end_title(self):
+ value = self.popContent('title')
+ context = self._getContext()
+ if self.intextinput:
+ context['textinput']['title'] = value
+ elif self.inimage:
+ context['image']['title'] = value
+ _end_dc_title = _end_title
+ _end_media_title = _end_title
+
+ def _start_description(self, attrsD):
+ context = self._getContext()
+ if context.has_key('summary'):
+ self._summaryKey = 'content'
+ self._start_content(attrsD)
+ else:
+ self.pushContent('description', attrsD, 'text/html', self.infeed or self.inentry or self.insource)
+
+ def _start_abstract(self, attrsD):
+ self.pushContent('description', attrsD, 'text/plain', self.infeed or self.inentry or self.insource)
+
+ def _end_description(self):
+ if self._summaryKey == 'content':
+ self._end_content()
+ else:
+ value = self.popContent('description')
+ context = self._getContext()
+ if self.intextinput:
+ context['textinput']['description'] = value
+ elif self.inimage:
+ context['image']['description'] = value
+ self._summaryKey = None
+ _end_abstract = _end_description
+
+ def _start_info(self, attrsD):
+ self.pushContent('info', attrsD, 'text/plain', 1)
+ _start_feedburner_browserfriendly = _start_info
+
+ def _end_info(self):
+ self.popContent('info')
+ _end_feedburner_browserfriendly = _end_info
+
+ def _start_generator(self, attrsD):
+ if attrsD:
+ attrsD = self._itsAnHrefDamnIt(attrsD)
+ if attrsD.has_key('href'):
+ attrsD['href'] = self.resolveURI(attrsD['href'])
+ self._getContext()['generator_detail'] = FeedParserDict(attrsD)
+ self.push('generator', 1)
+
+ def _end_generator(self):
+ value = self.pop('generator')
+ context = self._getContext()
+ if context.has_key('generator_detail'):
+ context['generator_detail']['name'] = value
+
+ def _start_admin_generatoragent(self, attrsD):
+ self.push('generator', 1)
+ value = self._getAttribute(attrsD, 'rdf:resource')
+ if value:
+ self.elementstack[-1][2].append(value)
+ self.pop('generator')
+ self._getContext()['generator_detail'] = FeedParserDict({'href': value})
+
+ def _start_admin_errorreportsto(self, attrsD):
+ self.push('errorreportsto', 1)
+ value = self._getAttribute(attrsD, 'rdf:resource')
+ if value:
+ self.elementstack[-1][2].append(value)
+ self.pop('errorreportsto')
+
+ def _start_summary(self, attrsD):
+ context = self._getContext()
+ if context.has_key('summary'):
+ self._summaryKey = 'content'
+ self._start_content(attrsD)
+ else:
+ self._summaryKey = 'summary'
+ self.pushContent(self._summaryKey, attrsD, 'text/plain', 1)
+ _start_itunes_summary = _start_summary
+
+ def _end_summary(self):
+ if self._summaryKey == 'content':
+ self._end_content()
+ else:
+ self.popContent(self._summaryKey or 'summary')
+ self._summaryKey = None
+ _end_itunes_summary = _end_summary
+
+ def _start_enclosure(self, attrsD):
+ attrsD = self._itsAnHrefDamnIt(attrsD)
+ self._getContext().setdefault('enclosures', []).append(FeedParserDict(attrsD))
+ href = attrsD.get('href')
+ if href:
+ context = self._getContext()
+ if not context.get('id'):
+ context['id'] = href
+
+ def _start_source(self, attrsD):
+ self.insource = 1
+
+ def _end_source(self):
+ self.insource = 0
+ self._getContext()['source'] = copy.deepcopy(self.sourcedata)
+ self.sourcedata.clear()
+
+ def _start_content(self, attrsD):
+ self.pushContent('content', attrsD, 'text/plain', 1)
+ src = attrsD.get('src')
+ if src:
+ self.contentparams['src'] = src
+ self.push('content', 1)
+
+ def _start_prodlink(self, attrsD):
+ self.pushContent('content', attrsD, 'text/html', 1)
+
+ def _start_body(self, attrsD):
+ self.pushContent('content', attrsD, 'application/xhtml+xml', 1)
+ _start_xhtml_body = _start_body
+
+ def _start_content_encoded(self, attrsD):
+ self.pushContent('content', attrsD, 'text/html', 1)
+ _start_fullitem = _start_content_encoded
+
+ def _end_content(self):
+ copyToDescription = self.mapContentType(self.contentparams.get('type')) in (['text/plain'] + self.html_types)
+ value = self.popContent('content')
+ if copyToDescription:
+ self._save('description', value)
+ _end_body = _end_content
+ _end_xhtml_body = _end_content
+ _end_content_encoded = _end_content
+ _end_fullitem = _end_content
+ _end_prodlink = _end_content
+
+ def _start_itunes_image(self, attrsD):
+ self.push('itunes_image', 0)
+ self._getContext()['image'] = FeedParserDict({'href': attrsD.get('href')})
+ _start_itunes_link = _start_itunes_image
+
+ def _end_itunes_block(self):
+ value = self.pop('itunes_block', 0)
+ self._getContext()['itunes_block'] = (value == 'yes') and 1 or 0
+
+ def _end_itunes_explicit(self):
+ value = self.pop('itunes_explicit', 0)
+ self._getContext()['itunes_explicit'] = (value == 'yes') and 1 or 0
+
+if _XML_AVAILABLE:
+ class _StrictFeedParser(_FeedParserMixin, xml.sax.handler.ContentHandler):
+ def __init__(self, baseuri, baselang, encoding):
+ if _debug: sys.stderr.write('trying StrictFeedParser\n')
+ xml.sax.handler.ContentHandler.__init__(self)
+ _FeedParserMixin.__init__(self, baseuri, baselang, encoding)
+ self.bozo = 0
+ self.exc = None
+
+ def startPrefixMapping(self, prefix, uri):
+ self.trackNamespace(prefix, uri)
+
+ def startElementNS(self, name, qname, attrs):
+ namespace, localname = name
+ lowernamespace = str(namespace or '').lower()
+ if lowernamespace.find('backend.userland.com/rss') <> -1:
+ # match any backend.userland.com namespace
+ namespace = 'http://backend.userland.com/rss'
+ lowernamespace = namespace
+ if qname and qname.find(':') > 0:
+ givenprefix = qname.split(':')[0]
+ else:
+ givenprefix = None
+ prefix = self._matchnamespaces.get(lowernamespace, givenprefix)
+ if givenprefix and (prefix == None or (prefix == '' and lowernamespace == '')) and not self.namespacesInUse.has_key(givenprefix):
+ raise UndeclaredNamespace, "'%s' is not associated with a namespace" % givenprefix
+ if prefix:
+ localname = prefix + ':' + localname
+ localname = str(localname).lower()
+ if _debug: sys.stderr.write('startElementNS: qname = %s, namespace = %s, givenprefix = %s, prefix = %s, attrs = %s, localname = %s\n' % (qname, namespace, givenprefix, prefix, attrs.items(), localname))
+
+ # qname implementation is horribly broken in Python 2.1 (it
+ # doesn't report any), and slightly broken in Python 2.2 (it
+ # doesn't report the xml: namespace). So we match up namespaces
+ # with a known list first, and then possibly override them with
+ # the qnames the SAX parser gives us (if indeed it gives us any
+ # at all). Thanks to MatejC for helping me test this and
+ # tirelessly telling me that it didn't work yet.
+ attrsD = {}
+ for (namespace, attrlocalname), attrvalue in attrs._attrs.items():
+ lowernamespace = (namespace or '').lower()
+ prefix = self._matchnamespaces.get(lowernamespace, '')
+ if prefix:
+ attrlocalname = prefix + ':' + attrlocalname
+ attrsD[str(attrlocalname).lower()] = attrvalue
+ for qname in attrs.getQNames():
+ attrsD[str(qname).lower()] = attrs.getValueByQName(qname)
+ self.unknown_starttag(localname, attrsD.items())
+
+ def characters(self, text):
+ self.handle_data(text)
+
+ def endElementNS(self, name, qname):
+ namespace, localname = name
+ lowernamespace = str(namespace or '').lower()
+ if qname and qname.find(':') > 0:
+ givenprefix = qname.split(':')[0]
+ else:
+ givenprefix = ''
+ prefix = self._matchnamespaces.get(lowernamespace, givenprefix)
+ if prefix:
+ localname = prefix + ':' + localname
+ localname = str(localname).lower()
+ self.unknown_endtag(localname)
+
+ def error(self, exc):
+ self.bozo = 1
+ self.exc = exc
+
+ def fatalError(self, exc):
+ self.error(exc)
+ raise exc
+
+class _BaseHTMLProcessor(sgmllib.SGMLParser):
+ elements_no_end_tag = ['area', 'base', 'basefont', 'br', 'col', 'frame', 'hr',
+ 'img', 'input', 'isindex', 'link', 'meta', 'param']
+
+ def __init__(self, encoding):
+ self.encoding = encoding
+ if _debug: sys.stderr.write('entering BaseHTMLProcessor, encoding=%s\n' % self.encoding)
+ sgmllib.SGMLParser.__init__(self)
+
+ def reset(self):
+ self.pieces = []
+ sgmllib.SGMLParser.reset(self)
+
+ def _shorttag_replace(self, match):
+ tag = match.group(1)
+ if tag in self.elements_no_end_tag:
+ return '<' + tag + ' />'
+ else:
+ return '<' + tag + '>'
+
+ def feed(self, data):
+ data = re.compile(r'', self._shorttag_replace, data) # bug [ 1399464 ] Bad regexp for _shorttag_replace
+ data = re.sub(r'<([^<\s]+?)\s*/>', self._shorttag_replace, data)
+ data = data.replace(''', "'")
+ data = data.replace('"', '"')
+ if self.encoding and type(data) == type(u''):
+ data = data.encode(self.encoding)
+ sgmllib.SGMLParser.feed(self, data)
+ sgmllib.SGMLParser.close(self)
+
+ def normalize_attrs(self, attrs):
+ # utility method to be called by descendants
+ attrs = [(k.lower(), v) for k, v in attrs]
+ attrs = [(k, k in ('rel', 'type') and v.lower() or v) for k, v in attrs]
+ return attrs
+
+ def unknown_starttag(self, tag, attrs):
+ # called for each start tag
+ # attrs is a list of (attr, value) tuples
+ # e.g. for '):
+ data = data.split('>', 1)[1]
+ if data.count(' = '2.3.3'
+ assert base64 != None
+ user, passw = base64.decodestring(req.headers['Authorization'].split(' ')[1]).split(':')
+ realm = re.findall('realm="([^"]*)"', headers['WWW-Authenticate'])[0]
+ self.add_password(realm, host, user, passw)
+ retry = self.http_error_auth_reqed('www-authenticate', host, req, headers)
+ self.reset_retry_count()
+ return retry
+ except:
+ return self.http_error_default(req, fp, code, msg, headers)
+
+def _open_resource(url_file_stream_or_string, etag, modified, agent, referrer, handlers):
+ """URL, filename, or string --> stream
+
+ This function lets you define parsers that take any input source
+ (URL, pathname to local or network file, or actual data as a string)
+ and deal with it in a uniform manner. Returned object is guaranteed
+ to have all the basic stdio read methods (read, readline, readlines).
+ Just .close() the object when you're done with it.
+
+ If the etag argument is supplied, it will be used as the value of an
+ If-None-Match request header.
+
+ If the modified argument is supplied, it must be a tuple of 9 integers
+ as returned by gmtime() in the standard Python time module. This MUST
+ be in GMT (Greenwich Mean Time). The formatted date/time will be used
+ as the value of an If-Modified-Since request header.
+
+ If the agent argument is supplied, it will be used as the value of a
+ User-Agent request header.
+
+ If the referrer argument is supplied, it will be used as the value of a
+ Referer[sic] request header.
+
+ If handlers is supplied, it is a list of handlers used to build a
+ urllib2 opener.
+ """
+
+ if hasattr(url_file_stream_or_string, 'read'):
+ return url_file_stream_or_string
+
+ if url_file_stream_or_string == '-':
+ return sys.stdin
+
+ if urlparse.urlparse(url_file_stream_or_string)[0] in ('http', 'https', 'ftp'):
+ if not agent:
+ agent = USER_AGENT
+ # test for inline user:password for basic auth
+ auth = None
+ if base64:
+ urltype, rest = urllib.splittype(url_file_stream_or_string)
+ realhost, rest = urllib.splithost(rest)
+ if realhost:
+ user_passwd, realhost = urllib.splituser(realhost)
+ if user_passwd:
+ url_file_stream_or_string = '%s://%s%s' % (urltype, realhost, rest)
+ auth = base64.encodestring(user_passwd).strip()
+ # try to open with urllib2 (to use optional headers)
+ request = urllib2.Request(url_file_stream_or_string)
+ request.add_header('User-Agent', agent)
+ if etag:
+ request.add_header('If-None-Match', etag)
+ if modified:
+ # format into an RFC 1123-compliant timestamp. We can't use
+ # time.strftime() since the %a and %b directives can be affected
+ # by the current locale, but RFC 2616 states that dates must be
+ # in English.
+ short_weekdays = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
+ months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
+ request.add_header('If-Modified-Since', '%s, %02d %s %04d %02d:%02d:%02d GMT' % (short_weekdays[modified[6]], modified[2], months[modified[1] - 1], modified[0], modified[3], modified[4], modified[5]))
+ if referrer:
+ request.add_header('Referer', referrer)
+ if gzip and zlib:
+ request.add_header('Accept-encoding', 'gzip, deflate')
+ elif gzip:
+ request.add_header('Accept-encoding', 'gzip')
+ elif zlib:
+ request.add_header('Accept-encoding', 'deflate')
+ else:
+ request.add_header('Accept-encoding', '')
+ if auth:
+ request.add_header('Authorization', 'Basic %s' % auth)
+ if ACCEPT_HEADER:
+ request.add_header('Accept', ACCEPT_HEADER)
+ request.add_header('A-IM', 'feed') # RFC 3229 support
+ opener = apply(urllib2.build_opener, tuple([_FeedURLHandler()] + handlers))
+ opener.addheaders = [] # RMK - must clear so we only send our custom User-Agent
+ try:
+ return opener.open(request)
+ finally:
+ opener.close() # JohnD
+
+ # try to open with native open function (if url_file_stream_or_string is a filename)
+ try:
+ return open(url_file_stream_or_string)
+ except:
+ pass
+
+ # treat url_file_stream_or_string as string
+ return _StringIO(str(url_file_stream_or_string))
+
+_date_handlers = []
+def registerDateHandler(func):
+ '''Register a date handler function (takes string, returns 9-tuple date in GMT)'''
+ _date_handlers.insert(0, func)
+
+# ISO-8601 date parsing routines written by Fazal Majid.
+# The ISO 8601 standard is very convoluted and irregular - a full ISO 8601
+# parser is beyond the scope of feedparser and would be a worthwhile addition
+# to the Python library.
+# A single regular expression cannot parse ISO 8601 date formats into groups
+# as the standard is highly irregular (for instance is 030104 2003-01-04 or
+# 0301-04-01), so we use templates instead.
+# Please note the order in templates is significant because we need a
+# greedy match.
+_iso8601_tmpl = ['YYYY-?MM-?DD', 'YYYY-MM', 'YYYY-?OOO',
+ 'YY-?MM-?DD', 'YY-?OOO', 'YYYY',
+ '-YY-?MM', '-OOO', '-YY',
+ '--MM-?DD', '--MM',
+ '---DD',
+ 'CC', '']
+_iso8601_re = [
+ tmpl.replace(
+ 'YYYY', r'(?P\d{4})').replace(
+ 'YY', r'(?P\d\d)').replace(
+ 'MM', r'(?P[01]\d)').replace(
+ 'DD', r'(?P[0123]\d)').replace(
+ 'OOO', r'(?P[0123]\d\d)').replace(
+ 'CC', r'(?P\d\d$)')
+ + r'(T?(?P\d{2}):(?P\d{2})'
+ + r'(:(?P\d{2}))?'
+ + r'(?P[+-](?P\d{2})(:(?P\d{2}))?|Z)?)?'
+ for tmpl in _iso8601_tmpl]
+del tmpl
+_iso8601_matches = [re.compile(regex).match for regex in _iso8601_re]
+del regex
+def _parse_date_iso8601(dateString):
+ '''Parse a variety of ISO-8601-compatible formats like 20040105'''
+ m = None
+ for _iso8601_match in _iso8601_matches:
+ m = _iso8601_match(dateString)
+ if m: break
+ if not m: return
+ if m.span() == (0, 0): return
+ params = m.groupdict()
+ ordinal = params.get('ordinal', 0)
+ if ordinal:
+ ordinal = int(ordinal)
+ else:
+ ordinal = 0
+ year = params.get('year', '--')
+ if not year or year == '--':
+ year = time.gmtime()[0]
+ elif len(year) == 2:
+ # ISO 8601 assumes current century, i.e. 93 -> 2093, NOT 1993
+ year = 100 * int(time.gmtime()[0] / 100) + int(year)
+ else:
+ year = int(year)
+ month = params.get('month', '-')
+ if not month or month == '-':
+ # ordinals are NOT normalized by mktime, we simulate them
+ # by setting month=1, day=ordinal
+ if ordinal:
+ month = 1
+ else:
+ month = time.gmtime()[1]
+ month = int(month)
+ day = params.get('day', 0)
+ if not day:
+ # see above
+ if ordinal:
+ day = ordinal
+ elif params.get('century', 0) or \
+ params.get('year', 0) or params.get('month', 0):
+ day = 1
+ else:
+ day = time.gmtime()[2]
+ else:
+ day = int(day)
+ # special case of the century - is the first year of the 21st century
+ # 2000 or 2001 ? The debate goes on...
+ if 'century' in params.keys():
+ year = (int(params['century']) - 1) * 100 + 1
+ # in ISO 8601 most fields are optional
+ for field in ['hour', 'minute', 'second', 'tzhour', 'tzmin']:
+ if not params.get(field, None):
+ params[field] = 0
+ hour = int(params.get('hour', 0))
+ minute = int(params.get('minute', 0))
+ second = int(params.get('second', 0))
+ # weekday is normalized by mktime(), we can ignore it
+ weekday = 0
+ # daylight savings is complex, but not needed for feedparser's purposes
+ # as time zones, if specified, include mention of whether it is active
+ # (e.g. PST vs. PDT, CET). Using -1 is implementation-dependent and
+ # and most implementations have DST bugs
+ daylight_savings_flag = 0
+ tm = [year, month, day, hour, minute, second, weekday,
+ ordinal, daylight_savings_flag]
+ # ISO 8601 time zone adjustments
+ tz = params.get('tz')
+ if tz and tz != 'Z':
+ if tz[0] == '-':
+ tm[3] += int(params.get('tzhour', 0))
+ tm[4] += int(params.get('tzmin', 0))
+ elif tz[0] == '+':
+ tm[3] -= int(params.get('tzhour', 0))
+ tm[4] -= int(params.get('tzmin', 0))
+ else:
+ return None
+ # Python's time.mktime() is a wrapper around the ANSI C mktime(3c)
+ # which is guaranteed to normalize d/m/y/h/m/s.
+ # Many implementations have bugs, but we'll pretend they don't.
+ return time.localtime(time.mktime(tm))
+registerDateHandler(_parse_date_iso8601)
+
+# 8-bit date handling routines written by ytrewq1.
+_korean_year = u'\ub144' # b3e2 in euc-kr
+_korean_month = u'\uc6d4' # bff9 in euc-kr
+_korean_day = u'\uc77c' # c0cf in euc-kr
+_korean_am = u'\uc624\uc804' # bfc0 c0fc in euc-kr
+_korean_pm = u'\uc624\ud6c4' # bfc0 c8c4 in euc-kr
+
+_korean_onblog_date_re = \
+ re.compile('(\d{4})%s\s+(\d{2})%s\s+(\d{2})%s\s+(\d{2}):(\d{2}):(\d{2})' % \
+ (_korean_year, _korean_month, _korean_day))
+_korean_nate_date_re = \
+ re.compile(u'(\d{4})-(\d{2})-(\d{2})\s+(%s|%s)\s+(\d{,2}):(\d{,2}):(\d{,2})' % \
+ (_korean_am, _korean_pm))
+def _parse_date_onblog(dateString):
+ '''Parse a string according to the OnBlog 8-bit date format'''
+ m = _korean_onblog_date_re.match(dateString)
+ if not m: return
+ w3dtfdate = '%(year)s-%(month)s-%(day)sT%(hour)s:%(minute)s:%(second)s%(zonediff)s' % \
+ {'year': m.group(1), 'month': m.group(2), 'day': m.group(3),\
+ 'hour': m.group(4), 'minute': m.group(5), 'second': m.group(6),\
+ 'zonediff': '+09:00'}
+ if _debug: sys.stderr.write('OnBlog date parsed as: %s\n' % w3dtfdate)
+ return _parse_date_w3dtf(w3dtfdate)
+registerDateHandler(_parse_date_onblog)
+
+def _parse_date_nate(dateString):
+ '''Parse a string according to the Nate 8-bit date format'''
+ m = _korean_nate_date_re.match(dateString)
+ if not m: return
+ hour = int(m.group(5))
+ ampm = m.group(4)
+ if (ampm == _korean_pm):
+ hour += 12
+ hour = str(hour)
+ if len(hour) == 1:
+ hour = '0' + hour
+ w3dtfdate = '%(year)s-%(month)s-%(day)sT%(hour)s:%(minute)s:%(second)s%(zonediff)s' % \
+ {'year': m.group(1), 'month': m.group(2), 'day': m.group(3),\
+ 'hour': hour, 'minute': m.group(6), 'second': m.group(7),\
+ 'zonediff': '+09:00'}
+ if _debug: sys.stderr.write('Nate date parsed as: %s\n' % w3dtfdate)
+ return _parse_date_w3dtf(w3dtfdate)
+registerDateHandler(_parse_date_nate)
+
+_mssql_date_re = \
+ re.compile('(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})(\.\d+)?')
+def _parse_date_mssql(dateString):
+ '''Parse a string according to the MS SQL date format'''
+ m = _mssql_date_re.match(dateString)
+ if not m: return
+ w3dtfdate = '%(year)s-%(month)s-%(day)sT%(hour)s:%(minute)s:%(second)s%(zonediff)s' % \
+ {'year': m.group(1), 'month': m.group(2), 'day': m.group(3),\
+ 'hour': m.group(4), 'minute': m.group(5), 'second': m.group(6),\
+ 'zonediff': '+09:00'}
+ if _debug: sys.stderr.write('MS SQL date parsed as: %s\n' % w3dtfdate)
+ return _parse_date_w3dtf(w3dtfdate)
+registerDateHandler(_parse_date_mssql)
+
+# Unicode strings for Greek date strings
+_greek_months = \
+ { \
+ u'\u0399\u03b1\u03bd': u'Jan', # c9e1ed in iso-8859-7
+ u'\u03a6\u03b5\u03b2': u'Feb', # d6e5e2 in iso-8859-7
+ u'\u039c\u03ac\u03ce': u'Mar', # ccdcfe in iso-8859-7
+ u'\u039c\u03b1\u03ce': u'Mar', # cce1fe in iso-8859-7
+ u'\u0391\u03c0\u03c1': u'Apr', # c1f0f1 in iso-8859-7
+ u'\u039c\u03ac\u03b9': u'May', # ccdce9 in iso-8859-7
+ u'\u039c\u03b1\u03ca': u'May', # cce1fa in iso-8859-7
+ u'\u039c\u03b1\u03b9': u'May', # cce1e9 in iso-8859-7
+ u'\u0399\u03bf\u03cd\u03bd': u'Jun', # c9effded in iso-8859-7
+ u'\u0399\u03bf\u03bd': u'Jun', # c9efed in iso-8859-7
+ u'\u0399\u03bf\u03cd\u03bb': u'Jul', # c9effdeb in iso-8859-7
+ u'\u0399\u03bf\u03bb': u'Jul', # c9f9eb in iso-8859-7
+ u'\u0391\u03cd\u03b3': u'Aug', # c1fde3 in iso-8859-7
+ u'\u0391\u03c5\u03b3': u'Aug', # c1f5e3 in iso-8859-7
+ u'\u03a3\u03b5\u03c0': u'Sep', # d3e5f0 in iso-8859-7
+ u'\u039f\u03ba\u03c4': u'Oct', # cfeaf4 in iso-8859-7
+ u'\u039d\u03bf\u03ad': u'Nov', # cdefdd in iso-8859-7
+ u'\u039d\u03bf\u03b5': u'Nov', # cdefe5 in iso-8859-7
+ u'\u0394\u03b5\u03ba': u'Dec', # c4e5ea in iso-8859-7
+ }
+
+_greek_wdays = \
+ { \
+ u'\u039a\u03c5\u03c1': u'Sun', # caf5f1 in iso-8859-7
+ u'\u0394\u03b5\u03c5': u'Mon', # c4e5f5 in iso-8859-7
+ u'\u03a4\u03c1\u03b9': u'Tue', # d4f1e9 in iso-8859-7
+ u'\u03a4\u03b5\u03c4': u'Wed', # d4e5f4 in iso-8859-7
+ u'\u03a0\u03b5\u03bc': u'Thu', # d0e5ec in iso-8859-7
+ u'\u03a0\u03b1\u03c1': u'Fri', # d0e1f1 in iso-8859-7
+ u'\u03a3\u03b1\u03b2': u'Sat', # d3e1e2 in iso-8859-7
+ }
+
+_greek_date_format_re = \
+ re.compile(u'([^,]+),\s+(\d{2})\s+([^\s]+)\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})\s+([^\s]+)')
+
+def _parse_date_greek(dateString):
+ '''Parse a string according to a Greek 8-bit date format.'''
+ m = _greek_date_format_re.match(dateString)
+ if not m: return
+ try:
+ wday = _greek_wdays[m.group(1)]
+ month = _greek_months[m.group(3)]
+ except:
+ return
+ rfc822date = '%(wday)s, %(day)s %(month)s %(year)s %(hour)s:%(minute)s:%(second)s %(zonediff)s' % \
+ {'wday': wday, 'day': m.group(2), 'month': month, 'year': m.group(4),\
+ 'hour': m.group(5), 'minute': m.group(6), 'second': m.group(7),\
+ 'zonediff': m.group(8)}
+ if _debug: sys.stderr.write('Greek date parsed as: %s\n' % rfc822date)
+ return _parse_date_rfc822(rfc822date)
+registerDateHandler(_parse_date_greek)
+
+# Unicode strings for Hungarian date strings
+_hungarian_months = \
+ { \
+ u'janu\u00e1r': u'01', # e1 in iso-8859-2
+ u'febru\u00e1ri': u'02', # e1 in iso-8859-2
+ u'm\u00e1rcius': u'03', # e1 in iso-8859-2
+ u'\u00e1prilis': u'04', # e1 in iso-8859-2
+ u'm\u00e1ujus': u'05', # e1 in iso-8859-2
+ u'j\u00fanius': u'06', # fa in iso-8859-2
+ u'j\u00falius': u'07', # fa in iso-8859-2
+ u'augusztus': u'08',
+ u'szeptember': u'09',
+ u'okt\u00f3ber': u'10', # f3 in iso-8859-2
+ u'november': u'11',
+ u'december': u'12',
+ }
+
+_hungarian_date_format_re = \
+ re.compile(u'(\d{4})-([^-]+)-(\d{,2})T(\d{,2}):(\d{2})((\+|-)(\d{,2}:\d{2}))')
+
+def _parse_date_hungarian(dateString):
+ '''Parse a string according to a Hungarian 8-bit date format.'''
+ m = _hungarian_date_format_re.match(dateString)
+ if not m: return
+ try:
+ month = _hungarian_months[m.group(2)]
+ day = m.group(3)
+ if len(day) == 1:
+ day = '0' + day
+ hour = m.group(4)
+ if len(hour) == 1:
+ hour = '0' + hour
+ except:
+ return
+ w3dtfdate = '%(year)s-%(month)s-%(day)sT%(hour)s:%(minute)s%(zonediff)s' % \
+ {'year': m.group(1), 'month': month, 'day': day,\
+ 'hour': hour, 'minute': m.group(5),\
+ 'zonediff': m.group(6)}
+ if _debug: sys.stderr.write('Hungarian date parsed as: %s\n' % w3dtfdate)
+ return _parse_date_w3dtf(w3dtfdate)
+registerDateHandler(_parse_date_hungarian)
+
+# W3DTF-style date parsing adapted from PyXML xml.utils.iso8601, written by
+# Drake and licensed under the Python license. Removed all range checking
+# for month, day, hour, minute, and second, since mktime will normalize
+# these later
+def _parse_date_w3dtf(dateString):
+ def __extract_date(m):
+ year = int(m.group('year'))
+ if year < 100:
+ year = 100 * int(time.gmtime()[0] / 100) + int(year)
+ if year < 1000:
+ return 0, 0, 0
+ julian = m.group('julian')
+ if julian:
+ julian = int(julian)
+ month = julian / 30 + 1
+ day = julian % 30 + 1
+ jday = None
+ while jday != julian:
+ t = time.mktime((year, month, day, 0, 0, 0, 0, 0, 0))
+ jday = time.gmtime(t)[-2]
+ diff = abs(jday - julian)
+ if jday > julian:
+ if diff < day:
+ day = day - diff
+ else:
+ month = month - 1
+ day = 31
+ elif jday < julian:
+ if day + diff < 28:
+ day = day + diff
+ else:
+ month = month + 1
+ return year, month, day
+ month = m.group('month')
+ day = 1
+ if month is None:
+ month = 1
+ else:
+ month = int(month)
+ day = m.group('day')
+ if day:
+ day = int(day)
+ else:
+ day = 1
+ return year, month, day
+
+ def __extract_time(m):
+ if not m:
+ return 0, 0, 0
+ hours = m.group('hours')
+ if not hours:
+ return 0, 0, 0
+ hours = int(hours)
+ minutes = int(m.group('minutes'))
+ seconds = m.group('seconds')
+ if seconds:
+ seconds = int(seconds)
+ else:
+ seconds = 0
+ return hours, minutes, seconds
+
+ def __extract_tzd(m):
+ '''Return the Time Zone Designator as an offset in seconds from UTC.'''
+ if not m:
+ return 0
+ tzd = m.group('tzd')
+ if not tzd:
+ return 0
+ if tzd == 'Z':
+ return 0
+ hours = int(m.group('tzdhours'))
+ minutes = m.group('tzdminutes')
+ if minutes:
+ minutes = int(minutes)
+ else:
+ minutes = 0
+ offset = (hours*60 + minutes) * 60
+ if tzd[0] == '+':
+ return -offset
+ return offset
+
+ __date_re = ('(?P\d\d\d\d)'
+ '(?:(?P-|)'
+ '(?:(?P\d\d\d)'
+ '|(?P\d\d)(?:(?P=dsep)(?P\d\d))?))?')
+ __tzd_re = '(?P[-+](?P\d\d)(?::?(?P\d\d))|Z)'
+ __tzd_rx = re.compile(__tzd_re)
+ __time_re = ('(?P\d\d)(?P:|)(?P\d\d)'
+ '(?:(?P=tsep)(?P\d\d(?:[.,]\d+)?))?'
+ + __tzd_re)
+ __datetime_re = '%s(?:T%s)?' % (__date_re, __time_re)
+ __datetime_rx = re.compile(__datetime_re)
+ m = __datetime_rx.match(dateString)
+ if (m is None) or (m.group() != dateString): return
+ gmt = __extract_date(m) + __extract_time(m) + (0, 0, 0)
+ if gmt[0] == 0: return
+ return time.gmtime(time.mktime(gmt) + __extract_tzd(m) - time.timezone)
+registerDateHandler(_parse_date_w3dtf)
+
+def _parse_date_rfc822(dateString):
+ '''Parse an RFC822, RFC1123, RFC2822, or asctime-style date'''
+ data = dateString.split()
+ if data[0][-1] in (',', '.') or data[0].lower() in rfc822._daynames:
+ del data[0]
+ if len(data) == 4:
+ s = data[3]
+ i = s.find('+')
+ if i > 0:
+ data[3:] = [s[:i], s[i+1:]]
+ else:
+ data.append('')
+ dateString = " ".join(data)
+ if len(data) < 5:
+ dateString += ' 00:00:00 GMT'
+ tm = rfc822.parsedate_tz(dateString)
+ if tm:
+ return time.gmtime(rfc822.mktime_tz(tm))
+# rfc822.py defines several time zones, but we define some extra ones.
+# 'ET' is equivalent to 'EST', etc.
+_additional_timezones = {'AT': -400, 'ET': -500, 'CT': -600, 'MT': -700, 'PT': -800}
+rfc822._timezones.update(_additional_timezones)
+registerDateHandler(_parse_date_rfc822)
+
+def _parse_date(dateString):
+ '''Parses a variety of date formats into a 9-tuple in GMT'''
+ for handler in _date_handlers:
+ try:
+ date9tuple = handler(dateString)
+ if not date9tuple: continue
+ if len(date9tuple) != 9:
+ if _debug: sys.stderr.write('date handler function must return 9-tuple\n')
+ raise ValueError
+ map(int, date9tuple)
+ return date9tuple
+ except Exception, e:
+ if _debug: sys.stderr.write('%s raised %s\n' % (handler.__name__, repr(e)))
+ pass
+ return None
+
+def _getCharacterEncoding(http_headers, xml_data):
+ '''Get the character encoding of the XML document
+
+ http_headers is a dictionary
+ xml_data is a raw string (not Unicode)
+
+ This is so much trickier than it sounds, it's not even funny.
+ According to RFC 3023 ('XML Media Types'), if the HTTP Content-Type
+ is application/xml, application/*+xml,
+ application/xml-external-parsed-entity, or application/xml-dtd,
+ the encoding given in the charset parameter of the HTTP Content-Type
+ takes precedence over the encoding given in the XML prefix within the
+ document, and defaults to 'utf-8' if neither are specified. But, if
+ the HTTP Content-Type is text/xml, text/*+xml, or
+ text/xml-external-parsed-entity, the encoding given in the XML prefix
+ within the document is ALWAYS IGNORED and only the encoding given in
+ the charset parameter of the HTTP Content-Type header should be
+ respected, and it defaults to 'us-ascii' if not specified.
+
+ Furthermore, discussion on the atom-syntax mailing list with the
+ author of RFC 3023 leads me to the conclusion that any document
+ served with a Content-Type of text/* and no charset parameter
+ must be treated as us-ascii. (We now do this.) And also that it
+ must always be flagged as non-well-formed. (We now do this too.)
+
+ If Content-Type is unspecified (input was local file or non-HTTP source)
+ or unrecognized (server just got it totally wrong), then go by the
+ encoding given in the XML prefix of the document and default to
+ 'iso-8859-1' as per the HTTP specification (RFC 2616).
+
+ Then, assuming we didn't find a character encoding in the HTTP headers
+ (and the HTTP Content-type allowed us to look in the body), we need
+ to sniff the first few bytes of the XML data and try to determine
+ whether the encoding is ASCII-compatible. Section F of the XML
+ specification shows the way here:
+ http://www.w3.org/TR/REC-xml/#sec-guessing-no-ext-info
+
+ If the sniffed encoding is not ASCII-compatible, we need to make it
+ ASCII compatible so that we can sniff further into the XML declaration
+ to find the encoding attribute, which will tell us the true encoding.
+
+ Of course, none of this guarantees that we will be able to parse the
+ feed in the declared character encoding (assuming it was declared
+ correctly, which many are not). CJKCodecs and iconv_codec help a lot;
+ you should definitely install them if you can.
+ http://cjkpython.i18n.org/
+ '''
+
+ def _parseHTTPContentType(content_type):
+ '''takes HTTP Content-Type header and returns (content type, charset)
+
+ If no charset is specified, returns (content type, '')
+ If no content type is specified, returns ('', '')
+ Both return parameters are guaranteed to be lowercase strings
+ '''
+ content_type = content_type or ''
+ content_type, params = cgi.parse_header(content_type)
+ return content_type, params.get('charset', '').replace("'", '')
+
+ sniffed_xml_encoding = ''
+ xml_encoding = ''
+ true_encoding = ''
+ http_content_type, http_encoding = _parseHTTPContentType(http_headers.get('content-type'))
+ # Must sniff for non-ASCII-compatible character encodings before
+ # searching for XML declaration. This heuristic is defined in
+ # section F of the XML specification:
+ # http://www.w3.org/TR/REC-xml/#sec-guessing-no-ext-info
+ try:
+ if xml_data[:4] == '\x4c\x6f\xa7\x94':
+ # EBCDIC
+ xml_data = _ebcdic_to_ascii(xml_data)
+ elif xml_data[:4] == '\x00\x3c\x00\x3f':
+ # UTF-16BE
+ sniffed_xml_encoding = 'utf-16be'
+ xml_data = unicode(xml_data, 'utf-16be').encode('utf-8')
+ elif (len(xml_data) >= 4) and (xml_data[:2] == '\xfe\xff') and (xml_data[2:4] != '\x00\x00'):
+ # UTF-16BE with BOM
+ sniffed_xml_encoding = 'utf-16be'
+ xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8')
+ elif xml_data[:4] == '\x3c\x00\x3f\x00':
+ # UTF-16LE
+ sniffed_xml_encoding = 'utf-16le'
+ xml_data = unicode(xml_data, 'utf-16le').encode('utf-8')
+ elif (len(xml_data) >= 4) and (xml_data[:2] == '\xff\xfe') and (xml_data[2:4] != '\x00\x00'):
+ # UTF-16LE with BOM
+ sniffed_xml_encoding = 'utf-16le'
+ xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8')
+ elif xml_data[:4] == '\x00\x00\x00\x3c':
+ # UTF-32BE
+ sniffed_xml_encoding = 'utf-32be'
+ xml_data = unicode(xml_data, 'utf-32be').encode('utf-8')
+ elif xml_data[:4] == '\x3c\x00\x00\x00':
+ # UTF-32LE
+ sniffed_xml_encoding = 'utf-32le'
+ xml_data = unicode(xml_data, 'utf-32le').encode('utf-8')
+ elif xml_data[:4] == '\x00\x00\xfe\xff':
+ # UTF-32BE with BOM
+ sniffed_xml_encoding = 'utf-32be'
+ xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8')
+ elif xml_data[:4] == '\xff\xfe\x00\x00':
+ # UTF-32LE with BOM
+ sniffed_xml_encoding = 'utf-32le'
+ xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8')
+ elif xml_data[:3] == '\xef\xbb\xbf':
+ # UTF-8 with BOM
+ sniffed_xml_encoding = 'utf-8'
+ xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8')
+ else:
+ # ASCII-compatible
+ pass
+ xml_encoding_match = re.compile('^<\?.*encoding=[\'"](.*?)[\'"].*\?>').match(xml_data)
+ except:
+ xml_encoding_match = None
+ if xml_encoding_match:
+ xml_encoding = xml_encoding_match.groups()[0].lower()
+ if sniffed_xml_encoding and (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode', 'iso-10646-ucs-4', 'ucs-4', 'csucs4', 'utf-16', 'utf-32', 'utf_16', 'utf_32', 'utf16', 'u16')):
+ xml_encoding = sniffed_xml_encoding
+ acceptable_content_type = 0
+ application_content_types = ('application/xml', 'application/xml-dtd', 'application/xml-external-parsed-entity')
+ text_content_types = ('text/xml', 'text/xml-external-parsed-entity')
+ if (http_content_type in application_content_types) or \
+ (http_content_type.startswith('application/') and http_content_type.endswith('+xml')):
+ acceptable_content_type = 1
+ true_encoding = http_encoding or xml_encoding or 'utf-8'
+ elif (http_content_type in text_content_types) or \
+ (http_content_type.startswith('text/')) and http_content_type.endswith('+xml'):
+ acceptable_content_type = 1
+ true_encoding = http_encoding or 'us-ascii'
+ elif http_content_type.startswith('text/'):
+ true_encoding = http_encoding or 'us-ascii'
+ elif http_headers and (not http_headers.has_key('content-type')):
+ true_encoding = xml_encoding or 'iso-8859-1'
+ else:
+ true_encoding = xml_encoding or 'utf-8'
+ return true_encoding, http_encoding, xml_encoding, sniffed_xml_encoding, acceptable_content_type
+
+def _toUTF8(data, encoding):
+ '''Changes an XML data stream on the fly to specify a new encoding
+
+ data is a raw sequence of bytes (not Unicode) that is presumed to be in %encoding already
+ encoding is a string recognized by encodings.aliases
+ '''
+ if _debug: sys.stderr.write('entering _toUTF8, trying encoding %s\n' % encoding)
+ # strip Byte Order Mark (if present)
+ if (len(data) >= 4) and (data[:2] == '\xfe\xff') and (data[2:4] != '\x00\x00'):
+ if _debug:
+ sys.stderr.write('stripping BOM\n')
+ if encoding != 'utf-16be':
+ sys.stderr.write('trying utf-16be instead\n')
+ encoding = 'utf-16be'
+ data = data[2:]
+ elif (len(data) >= 4) and (data[:2] == '\xff\xfe') and (data[2:4] != '\x00\x00'):
+ if _debug:
+ sys.stderr.write('stripping BOM\n')
+ if encoding != 'utf-16le':
+ sys.stderr.write('trying utf-16le instead\n')
+ encoding = 'utf-16le'
+ data = data[2:]
+ elif data[:3] == '\xef\xbb\xbf':
+ if _debug:
+ sys.stderr.write('stripping BOM\n')
+ if encoding != 'utf-8':
+ sys.stderr.write('trying utf-8 instead\n')
+ encoding = 'utf-8'
+ data = data[3:]
+ elif data[:4] == '\x00\x00\xfe\xff':
+ if _debug:
+ sys.stderr.write('stripping BOM\n')
+ if encoding != 'utf-32be':
+ sys.stderr.write('trying utf-32be instead\n')
+ encoding = 'utf-32be'
+ data = data[4:]
+ elif data[:4] == '\xff\xfe\x00\x00':
+ if _debug:
+ sys.stderr.write('stripping BOM\n')
+ if encoding != 'utf-32le':
+ sys.stderr.write('trying utf-32le instead\n')
+ encoding = 'utf-32le'
+ data = data[4:]
+ newdata = unicode(data, encoding)
+ if _debug: sys.stderr.write('successfully converted %s data to unicode\n' % encoding)
+ declmatch = re.compile('^<\?xml[^>]*?>')
+ newdecl = ''''''
+ if declmatch.search(newdata):
+ newdata = declmatch.sub(newdecl, newdata)
+ else:
+ newdata = newdecl + u'\n' + newdata
+ return newdata.encode('utf-8')
+
+def _stripDoctype(data):
+ '''Strips DOCTYPE from XML document, returns (rss_version, stripped_data)
+
+ rss_version may be 'rss091n' or None
+ stripped_data is the same XML document, minus the DOCTYPE
+ '''
+ entity_pattern = re.compile(r']*?)>', re.MULTILINE)
+ data = entity_pattern.sub('', data)
+ doctype_pattern = re.compile(r']*?)>', re.MULTILINE)
+ doctype_results = doctype_pattern.findall(data)
+ doctype = doctype_results and doctype_results[0] or ''
+ if doctype.lower().count('netscape'):
+ version = 'rss091n'
+ else:
+ version = None
+ data = doctype_pattern.sub('', data)
+ return version, data
+
+def parse(url_file_stream_or_string, etag=None, modified=None, agent=None, referrer=None, handlers=[]):
+ '''Parse a feed from a URL, file, stream, or string'''
+ result = FeedParserDict()
+ result['feed'] = FeedParserDict()
+ result['entries'] = []
+ if _XML_AVAILABLE:
+ result['bozo'] = 0
+ if type(handlers) == types.InstanceType:
+ handlers = [handlers]
+ try:
+ f = _open_resource(url_file_stream_or_string, etag, modified, agent, referrer, handlers)
+ data = f.read()
+ except Exception, e:
+ result['bozo'] = 1
+ result['bozo_exception'] = e
+ data = ''
+ f = None
+
+ # if feed is gzip-compressed, decompress it
+ if f and data and hasattr(f, 'headers'):
+ if gzip and f.headers.get('content-encoding', '') == 'gzip':
+ try:
+ data = gzip.GzipFile(fileobj=_StringIO(data)).read()
+ except Exception, e:
+ # Some feeds claim to be gzipped but they're not, so
+ # we get garbage. Ideally, we should re-request the
+ # feed without the 'Accept-encoding: gzip' header,
+ # but we don't.
+ result['bozo'] = 1
+ result['bozo_exception'] = e
+ data = ''
+ elif zlib and f.headers.get('content-encoding', '') == 'deflate':
+ try:
+ data = zlib.decompress(data, -zlib.MAX_WBITS)
+ except Exception, e:
+ result['bozo'] = 1
+ result['bozo_exception'] = e
+ data = ''
+
+ # save HTTP headers
+ if hasattr(f, 'info'):
+ info = f.info()
+ result['etag'] = info.getheader('ETag')
+ last_modified = info.getheader('Last-Modified')
+ if last_modified:
+ result['modified'] = _parse_date(last_modified)
+ if hasattr(f, 'url'):
+ result['href'] = f.url
+ result['status'] = 200
+ if hasattr(f, 'status'):
+ result['status'] = f.status
+ if hasattr(f, 'headers'):
+ result['headers'] = f.headers.dict
+ if hasattr(f, 'close'):
+ f.close()
+
+ # there are four encodings to keep track of:
+ # - http_encoding is the encoding declared in the Content-Type HTTP header
+ # - xml_encoding is the encoding declared in the ; changed
+# project name
+#2.5 - 7/25/2003 - MAP - changed to Python license (all contributors agree);
+# removed unnecessary urllib code -- urllib2 should always be available anyway;
+# return actual url, status, and full HTTP headers (as result['url'],
+# result['status'], and result['headers']) if parsing a remote feed over HTTP --
+# this should pass all the HTTP tests at and as used in some RSS 2.0 feeds
+#2.5.3 - 8/6/2003 - TvdV - patch to track whether we're inside an image or
+# textInput, and also to return the character encoding (if specified)
+#2.6 - 1/1/2004 - MAP - dc:author support (MarekK); fixed bug tracking
+# nested divs within content (JohnD); fixed missing sys import (JohanS);
+# fixed regular expression to capture XML character encoding (Andrei);
+# added support for Atom 0.3-style links; fixed bug with textInput tracking;
+# added support for cloud (MartijnP); added support for multiple
+# category/dc:subject (MartijnP); normalize content model: 'description' gets
+# description (which can come from description, summary, or full content if no
+# description), 'content' gets dict of base/language/type/value (which can come
+# from content:encoded, xhtml:body, content, or fullitem);
+# fixed bug matching arbitrary Userland namespaces; added xml:base and xml:lang
+# tracking; fixed bug tracking unknown tags; fixed bug tracking content when
+# element is not in default namespace (like Pocketsoap feed);
+# resolve relative URLs in link, guid, docs, url, comments, wfw:comment,
+# wfw:commentRSS; resolve relative URLs within embedded HTML markup in
+# description, xhtml:body, content, content:encoded, title, subtitle,
+# summary, info, tagline, and copyright; added support for pingback and
+# trackback namespaces
+#2.7 - 1/5/2004 - MAP - really added support for trackback and pingback
+# namespaces, as opposed to 2.6 when I said I did but didn't really;
+# sanitize HTML markup within some elements; added mxTidy support (if
+# installed) to tidy HTML markup within some elements; fixed indentation
+# bug in _parse_date (FazalM); use socket.setdefaulttimeout if available
+# (FazalM); universal date parsing and normalization (FazalM): 'created', modified',
+# 'issued' are parsed into 9-tuple date format and stored in 'created_parsed',
+# 'modified_parsed', and 'issued_parsed'; 'date' is duplicated in 'modified'
+# and vice-versa; 'date_parsed' is duplicated in 'modified_parsed' and vice-versa
+#2.7.1 - 1/9/2004 - MAP - fixed bug handling " and '. fixed memory
+# leak not closing url opener (JohnD); added dc:publisher support (MarekK);
+# added admin:errorReportsTo support (MarekK); Python 2.1 dict support (MarekK)
+#2.7.4 - 1/14/2004 - MAP - added workaround for improperly formed
tags in +# encoded HTML (skadz); fixed unicode handling in normalize_attrs (ChrisL); +# fixed relative URI processing for guid (skadz); added ICBM support; added +# base64 support +#2.7.5 - 1/15/2004 - MAP - added workaround for malformed DOCTYPE (seen on many +# blogspot.com sites); added _debug variable +#2.7.6 - 1/16/2004 - MAP - fixed bug with StringIO importing +#3.0b3 - 1/23/2004 - MAP - parse entire feed with real XML parser (if available); +# added several new supported namespaces; fixed bug tracking naked markup in +# description; added support for enclosure; added support for source; re-added +# support for cloud which got dropped somehow; added support for expirationDate +#3.0b4 - 1/26/2004 - MAP - fixed xml:lang inheritance; fixed multiple bugs tracking +# xml:base URI, one for documents that don't define one explicitly and one for +# documents that define an outer and an inner xml:base that goes out of scope +# before the end of the document +#3.0b5 - 1/26/2004 - MAP - fixed bug parsing multiple links at feed level +#3.0b6 - 1/27/2004 - MAP - added feed type and version detection, result['version'] +# will be one of SUPPORTED_VERSIONS.keys() or empty string if unrecognized; +# added support for creativeCommons:license and cc:license; added support for +# full Atom content model in title, tagline, info, copyright, summary; fixed bug +# with gzip encoding (not always telling server we support it when we do) +#3.0b7 - 1/28/2004 - MAP - support Atom-style author element in author_detail +# (dictionary of 'name', 'url', 'email'); map author to author_detail if author +# contains name + email address +#3.0b8 - 1/28/2004 - MAP - added support for contributor +#3.0b9 - 1/29/2004 - MAP - fixed check for presence of dict function; added +# support for summary +#3.0b10 - 1/31/2004 - MAP - incorporated ISO-8601 date parsing routines from +# xml.util.iso8601 +#3.0b11 - 2/2/2004 - MAP - added 'rights' to list of elements that can contain +# dangerous markup; fiddled with decodeEntities (not right); liberalized +# date parsing even further +#3.0b12 - 2/6/2004 - MAP - fiddled with decodeEntities (still not right); +# added support to Atom 0.2 subtitle; added support for Atom content model +# in copyright; better sanitizing of dangerous HTML elements with end tags +# (script, frameset) +#3.0b13 - 2/8/2004 - MAP - better handling of empty HTML tags (br, hr, img, +# etc.) in embedded markup, in either HTML or XHTML form (
,
,
) +#3.0b14 - 2/8/2004 - MAP - fixed CDATA handling in non-wellformed feeds under +# Python 2.1 +#3.0b15 - 2/11/2004 - MAP - fixed bug resolving relative links in wfw:commentRSS; +# fixed bug capturing author and contributor URL; fixed bug resolving relative +# links in author and contributor URL; fixed bug resolvin relative links in +# generator URL; added support for recognizing RSS 1.0; passed Simon Fell's +# namespace tests, and included them permanently in the test suite with his +# permission; fixed namespace handling under Python 2.1 +#3.0b16 - 2/12/2004 - MAP - fixed support for RSS 0.90 (broken in b15) +#3.0b17 - 2/13/2004 - MAP - determine character encoding as per RFC 3023 +#3.0b18 - 2/17/2004 - MAP - always map description to summary_detail (Andrei); +# use libxml2 (if available) +#3.0b19 - 3/15/2004 - MAP - fixed bug exploding author information when author +# name was in parentheses; removed ultra-problematic mxTidy support; patch to +# workaround crash in PyXML/expat when encountering invalid entities +# (MarkMoraes); support for textinput/textInput +#3.0b20 - 4/7/2004 - MAP - added CDF support +#3.0b21 - 4/14/2004 - MAP - added Hot RSS support +#3.0b22 - 4/19/2004 - MAP - changed 'channel' to 'feed', 'item' to 'entries' in +# results dict; changed results dict to allow getting values with results.key +# as well as results[key]; work around embedded illformed HTML with half +# a DOCTYPE; work around malformed Content-Type header; if character encoding +# is wrong, try several common ones before falling back to regexes (if this +# works, bozo_exception is set to CharacterEncodingOverride); fixed character +# encoding issues in BaseHTMLProcessor by tracking encoding and converting +# from Unicode to raw strings before feeding data to sgmllib.SGMLParser; +# convert each value in results to Unicode (if possible), even if using +# regex-based parsing +#3.0b23 - 4/21/2004 - MAP - fixed UnicodeDecodeError for feeds that contain +# high-bit characters in attributes in embedded HTML in description (thanks +# Thijs van de Vossen); moved guid, date, and date_parsed to mapped keys in +# FeedParserDict; tweaked FeedParserDict.has_key to return True if asking +# about a mapped key +#3.0fc1 - 4/23/2004 - MAP - made results.entries[0].links[0] and +# results.entries[0].enclosures[0] into FeedParserDict; fixed typo that could +# cause the same encoding to be tried twice (even if it failed the first time); +# fixed DOCTYPE stripping when DOCTYPE contained entity declarations; +# better textinput and image tracking in illformed RSS 1.0 feeds +#3.0fc2 - 5/10/2004 - MAP - added and passed Sam's amp tests; added and passed +# my blink tag tests +#3.0fc3 - 6/18/2004 - MAP - fixed bug in _changeEncodingDeclaration that +# failed to parse utf-16 encoded feeds; made source into a FeedParserDict; +# duplicate admin:generatorAgent/@rdf:resource in generator_detail.url; +# added support for image; refactored parse() fallback logic to try other +# encodings if SAX parsing fails (previously it would only try other encodings +# if re-encoding failed); remove unichr madness in normalize_attrs now that +# we're properly tracking encoding in and out of BaseHTMLProcessor; set +# feed.language from root-level xml:lang; set entry.id from rdf:about; +# send Accept header +#3.0 - 6/21/2004 - MAP - don't try iso-8859-1 (can't distinguish between +# iso-8859-1 and windows-1252 anyway, and most incorrectly marked feeds are +# windows-1252); fixed regression that could cause the same encoding to be +# tried twice (even if it failed the first time) +#3.0.1 - 6/22/2004 - MAP - default to us-ascii for all text/* content types; +# recover from malformed content-type header parameter with no equals sign +# ('text/xml; charset:iso-8859-1') +#3.1 - 6/28/2004 - MAP - added and passed tests for converting HTML entities +# to Unicode equivalents in illformed feeds (aaronsw); added and +# passed tests for converting character entities to Unicode equivalents +# in illformed feeds (aaronsw); test for valid parsers when setting +# XML_AVAILABLE; make version and encoding available when server returns +# a 304; add handlers parameter to pass arbitrary urllib2 handlers (like +# digest auth or proxy support); add code to parse username/password +# out of url and send as basic authentication; expose downloading-related +# exceptions in bozo_exception (aaronsw); added __contains__ method to +# FeedParserDict (aaronsw); added publisher_detail (aaronsw) +#3.2 - 7/3/2004 - MAP - use cjkcodecs and iconv_codec if available; always +# convert feed to UTF-8 before passing to XML parser; completely revamped +# logic for determining character encoding and attempting XML parsing +# (much faster); increased default timeout to 20 seconds; test for presence +# of Location header on redirects; added tests for many alternate character +# encodings; support various EBCDIC encodings; support UTF-16BE and +# UTF16-LE with or without a BOM; support UTF-8 with a BOM; support +# UTF-32BE and UTF-32LE with or without a BOM; fixed crashing bug if no +# XML parsers are available; added support for 'Content-encoding: deflate'; +# send blank 'Accept-encoding: ' header if neither gzip nor zlib modules +# are available +#3.3 - 7/15/2004 - MAP - optimize EBCDIC to ASCII conversion; fix obscure +# problem tracking xml:base and xml:lang if element declares it, child +# doesn't, first grandchild redeclares it, and second grandchild doesn't; +# refactored date parsing; defined public registerDateHandler so callers +# can add support for additional date formats at runtime; added support +# for OnBlog, Nate, MSSQL, Greek, and Hungarian dates (ytrewq1); added +# zopeCompatibilityHack() which turns FeedParserDict into a regular +# dictionary, required for Zope compatibility, and also makes command- +# line debugging easier because pprint module formats real dictionaries +# better than dictionary-like objects; added NonXMLContentType exception, +# which is stored in bozo_exception when a feed is served with a non-XML +# media type such as 'text/plain'; respect Content-Language as default +# language if not xml:lang is present; cloud dict is now FeedParserDict; +# generator dict is now FeedParserDict; better tracking of xml:lang, +# including support for xml:lang='' to unset the current language; +# recognize RSS 1.0 feeds even when RSS 1.0 namespace is not the default +# namespace; don't overwrite final status on redirects (scenarios: +# redirecting to a URL that returns 304, redirecting to a URL that +# redirects to another URL with a different type of redirect); add +# support for HTTP 303 redirects +#4.0 - MAP - support for relative URIs in xml:base attribute; fixed +# encoding issue with mxTidy (phopkins); preliminary support for RFC 3229; +# support for Atom 1.0; support for iTunes extensions; new 'tags' for +# categories/keywords/etc. as array of dict +# {'term': term, 'scheme': scheme, 'label': label} to match Atom 1.0 +# terminology; parse RFC 822-style dates with no time; lots of other +# bug fixes +#4.1 - MAP - removed socket timeout; added support for chardet library diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/htmltmpl.py b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/htmltmpl.py new file mode 100755 index 0000000..be6e41b --- /dev/null +++ b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/htmltmpl.py @@ -0,0 +1,1480 @@ + +""" A templating engine for separation of code and HTML. + + The documentation of this templating engine is separated to two parts: + + 1. Description of the templating language. + + 2. Documentation of classes and API of this module that provides + a Python implementation of the templating language. + + All the documentation can be found in 'doc' directory of the + distribution tarball or at the homepage of the engine. + Latest versions of this module are also available at that website. + + You can use and redistribute this module under conditions of the + GNU General Public License that can be found either at + [ http://www.gnu.org/ ] or in file "LICENSE" contained in the + distribution tarball of this module. + + Copyright (c) 2001 Tomas Styblo, tripie@cpan.org + + @name htmltmpl + @version 1.22 + @author-name Tomas Styblo + @author-email tripie@cpan.org + @website http://htmltmpl.sourceforge.net/ + @license-name GNU GPL + @license-url http://www.gnu.org/licenses/gpl.html +""" + +__version__ = 1.22 +__author__ = "Tomas Styblo (tripie@cpan.org)" + +# All imported modules are part of the standard Python library. + +from types import * +import re +import os +import os.path +import pprint # only for debugging +import sys +import copy +import cgi # for HTML escaping of variables +import urllib # for URL escaping of variables +import cPickle # for template compilation +import gettext + +INCLUDE_DIR = "inc" + +# Total number of possible parameters. +# Increment if adding a parameter to any statement. +PARAMS_NUMBER = 3 + +# Relative positions of parameters in TemplateCompiler.tokenize(). +PARAM_NAME = 1 +PARAM_ESCAPE = 2 +PARAM_GLOBAL = 3 +PARAM_GETTEXT_STRING = 1 + +# Find a way to lock files. Currently implemented only for UNIX and windows. +LOCKTYPE_FCNTL = 1 +LOCKTYPE_MSVCRT = 2 +LOCKTYPE = None +try: + import fcntl +except: + try: + import msvcrt + except: + LOCKTYPE = None + else: + LOCKTYPE = LOCKTYPE_MSVCRT +else: + LOCKTYPE = LOCKTYPE_FCNTL +LOCK_EX = 1 +LOCK_SH = 2 +LOCK_UN = 3 + +############################################## +# CLASS: TemplateManager # +############################################## + +class TemplateManager: + """ Class that manages compilation and precompilation of templates. + + You should use this class whenever you work with templates + that are stored in a file. The class can create a compiled + template and transparently manage its precompilation. It also + keeps the precompiled templates up-to-date by modification times + comparisons. + """ + + def __init__(self, include=1, max_include=5, precompile=1, comments=1, + gettext=0, debug=0): + """ Constructor. + + @header + __init__(include=1, max_include=5, precompile=1, comments=1, + gettext=0, debug=0) + + @param include Enable or disable included templates. + This optional parameter can be used to enable or disable + TMPL_INCLUDE inclusion of templates. Disabling of + inclusion can improve performance a bit. The inclusion is + enabled by default. + + @param max_include Maximum depth of nested inclusions. + This optional parameter can be used to specify maximum depth of + nested TMPL_INCLUDE inclusions. It defaults to 5. + This setting prevents infinite recursive inclusions. + + @param precompile Enable or disable precompilation of templates. + This optional parameter can be used to enable or disable + creation and usage of precompiled templates. + + A precompiled template is saved to the same directory in + which the main template file is located. You need write + permissions to that directory. + + Precompilation provides a significant performance boost because + it's not necessary to parse the templates over and over again. + The boost is especially noticeable when templates that include + other templates are used. + + Comparison of modification times of the main template and all + included templates is used to ensure that the precompiled + templates are up-to-date. Templates are also recompiled if the + htmltmpl module is updated. + + The TemplateErrorexception is raised when the precompiled + template cannot be saved. Precompilation is enabled by default. + + Precompilation is available only on UNIX and Windows platforms, + because proper file locking which is necessary to ensure + multitask safe behaviour is platform specific and is not + implemented for other platforms. Attempts to enable precompilation + on the other platforms result in raise of the + TemplateError exception. + + @param comments Enable or disable template comments. + This optional parameter can be used to enable or disable + template comments. + Disabling of the comments can improve performance a bit. + Comments are enabled by default. + + @param gettext Enable or disable gettext support. + + @param debug Enable or disable debugging messages. + This optional parameter is a flag that can be used to enable + or disable debugging messages which are printed to the standard + error output. The debugging messages are disabled by default. + """ + # Save the optional parameters. + # These values are not modified by any method. + self._include = include + self._max_include = max_include + self._precompile = precompile + self._comments = comments + self._gettext = gettext + self._debug = debug + + # Find what module to use to lock files. + # File locking is necessary for the 'precompile' feature to be + # multitask/thread safe. Currently it works only on UNIX + # and Windows. Anyone willing to implement it on Mac ? + if precompile and not LOCKTYPE: + raise TemplateError, "Template precompilation is not "\ + "available on this platform." + self.DEB("INIT DONE") + + def prepare(self, file): + """ Preprocess, parse, tokenize and compile the template. + + If precompilation is enabled then this method tries to load + a precompiled form of the template from the same directory + in which the template source file is located. If it succeeds, + then it compares modification times stored in the precompiled + form to modification times of source files of the template, + including source files of all templates included via the + TMPL_INCLUDE statements. If any of the modification times + differs, then the template is recompiled and the precompiled + form updated. + + If precompilation is disabled, then this method parses and + compiles the template. + + @header prepare(file) + + @return Compiled template. + The methods returns an instance of the Template class + which is a compiled form of the template. This instance can be + used as input for the TemplateProcessor. + + @param file Path to the template file to prepare. + The method looks for the template file in current directory + if the parameter is a relative path. All included templates must + be placed in subdirectory 'inc' of the + directory in which the main template file is located. + """ + compiled = None + if self._precompile: + if self.is_precompiled(file): + try: + precompiled = self.load_precompiled(file) + except PrecompiledError, template: + print >> sys.stderr, "Htmltmpl: bad precompiled "\ + "template '%s' removed" % template + compiled = self.compile(file) + self.save_precompiled(compiled) + else: + precompiled.debug(self._debug) + compile_params = (self._include, self._max_include, + self._comments, self._gettext) + if precompiled.is_uptodate(compile_params): + self.DEB("PRECOMPILED: UPTODATE") + compiled = precompiled + else: + self.DEB("PRECOMPILED: NOT UPTODATE") + compiled = self.update(precompiled) + else: + self.DEB("PRECOMPILED: NOT PRECOMPILED") + compiled = self.compile(file) + self.save_precompiled(compiled) + else: + self.DEB("PRECOMPILATION DISABLED") + compiled = self.compile(file) + return compiled + + def update(self, template): + """ Update (recompile) a compiled template. + + This method recompiles a template compiled from a file. + If precompilation is enabled then the precompiled form saved on + disk is also updated. + + @header update(template) + + @return Recompiled template. + It's ensured that the returned template is up-to-date. + + @param template A compiled template. + This parameter should be an instance of the Template + class, created either by the TemplateManager or by the + TemplateCompiler. The instance must represent a template + compiled from a file on disk. + """ + self.DEB("UPDATE") + updated = self.compile(template.file()) + if self._precompile: + self.save_precompiled(updated) + return updated + + ############################################## + # PRIVATE METHODS # + ############################################## + + def DEB(self, str): + """ Print debugging message to stderr if debugging is enabled. + @hidden + """ + if self._debug: print >> sys.stderr, str + + def lock_file(self, file, lock): + """ Provide platform independent file locking. + @hidden + """ + fd = file.fileno() + if LOCKTYPE == LOCKTYPE_FCNTL: + if lock == LOCK_SH: + fcntl.flock(fd, fcntl.LOCK_SH) + elif lock == LOCK_EX: + fcntl.flock(fd, fcntl.LOCK_EX) + elif lock == LOCK_UN: + fcntl.flock(fd, fcntl.LOCK_UN) + else: + raise TemplateError, "BUG: bad lock in lock_file" + elif LOCKTYPE == LOCKTYPE_MSVCRT: + if lock == LOCK_SH: + # msvcrt does not support shared locks :-( + msvcrt.locking(fd, msvcrt.LK_LOCK, 1) + elif lock == LOCK_EX: + msvcrt.locking(fd, msvcrt.LK_LOCK, 1) + elif lock == LOCK_UN: + msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) + else: + raise TemplateError, "BUG: bad lock in lock_file" + else: + raise TemplateError, "BUG: bad locktype in lock_file" + + def compile(self, file): + """ Compile the template. + @hidden + """ + return TemplateCompiler(self._include, self._max_include, + self._comments, self._gettext, + self._debug).compile(file) + + def is_precompiled(self, file): + """ Return true if the template is already precompiled on the disk. + This method doesn't check whether the compiled template is + uptodate. + @hidden + """ + filename = file + "c" # "template.tmplc" + if os.path.isfile(filename): + return 1 + else: + return 0 + + def load_precompiled(self, file): + """ Load precompiled template from disk. + + Remove the precompiled template file and recompile it + if the file contains corrupted or unpicklable data. + + @hidden + """ + filename = file + "c" # "template.tmplc" + self.DEB("LOADING PRECOMPILED") + try: + remove_bad = 0 + file = None + try: + file = open(filename, "rb") + self.lock_file(file, LOCK_SH) + precompiled = cPickle.load(file) + except IOError, (errno, errstr): + raise TemplateError, "IO error in load precompiled "\ + "template '%s': (%d) %s"\ + % (filename, errno, errstr) + except cPickle.UnpicklingError: + remove_bad = 1 + raise PrecompiledError, filename + except: + remove_bad = 1 + raise + else: + return precompiled + finally: + if file: + self.lock_file(file, LOCK_UN) + file.close() + if remove_bad and os.path.isfile(filename): + # X: We may lose the original exception here, raising OSError. + os.remove(filename) + + def save_precompiled(self, template): + """ Save compiled template to disk in precompiled form. + + Associated metadata is also saved. It includes: filename of the + main template file, modification time of the main template file, + modification times of all included templates and version of the + htmltmpl module which compiled the template. + + The method removes a file which is saved only partially because + of some error. + + @hidden + """ + filename = template.file() + "c" # creates "template.tmplc" + # Check if we have write permission to the template's directory. + template_dir = os.path.dirname(os.path.abspath(filename)) + if not os.access(template_dir, os.W_OK): + raise TemplateError, "Cannot save precompiled templates "\ + "to '%s': write permission denied."\ + % template_dir + try: + remove_bad = 0 + file = None + try: + file = open(filename, "wb") # may truncate existing file + self.lock_file(file, LOCK_EX) + BINARY = 1 + READABLE = 0 + if self._debug: + cPickle.dump(template, file, READABLE) + else: + cPickle.dump(template, file, BINARY) + except IOError, (errno, errstr): + remove_bad = 1 + raise TemplateError, "IO error while saving precompiled "\ + "template '%s': (%d) %s"\ + % (filename, errno, errstr) + except cPickle.PicklingError, error: + remove_bad = 1 + raise TemplateError, "Pickling error while saving "\ + "precompiled template '%s': %s"\ + % (filename, error) + except: + remove_bad = 1 + raise + else: + self.DEB("SAVING PRECOMPILED") + finally: + if file: + self.lock_file(file, LOCK_UN) + file.close() + if remove_bad and os.path.isfile(filename): + # X: We may lose the original exception here, raising OSError. + os.remove(filename) + + +############################################## +# CLASS: TemplateProcessor # +############################################## + +class TemplateProcessor: + """ Fill the template with data and process it. + + This class provides actual processing of a compiled template. + Use it to set template variables and loops and then obtain + result of the processing. + """ + + def __init__(self, html_escape=1, magic_vars=1, global_vars=0, debug=0): + """ Constructor. + + @header __init__(html_escape=1, magic_vars=1, global_vars=0, + debug=0) + + @param html_escape Enable or disable HTML escaping of variables. + This optional parameter is a flag that can be used to enable or + disable automatic HTML escaping of variables. + All variables are by default automatically HTML escaped. + The escaping process substitutes HTML brackets, ampersands and + double quotes with appropriate HTML entities. + + @param magic_vars Enable or disable loop magic variables. + This parameter can be used to enable or disable + "magic" context variables, that are automatically defined inside + loops. Magic variables are enabled by default. + + Refer to the language specification for description of these + magic variables. + + @param global_vars Globally activate global lookup of variables. + This optional parameter is a flag that can be used to specify + whether variables which cannot be found in the current scope + should be automatically looked up in enclosing scopes. + + Automatic global lookup is disabled by default. Global lookup + can be overriden on a per-variable basis by the + GLOBAL parameter of a TMPL_VAR + statement. + + @param debug Enable or disable debugging messages. + """ + self._html_escape = html_escape + self._magic_vars = magic_vars + self._global_vars = global_vars + self._debug = debug + + # Data structure containing variables and loops set by the + # application. Use debug=1, process some template and + # then check stderr to see how the structure looks. + # It's modified only by set() and reset() methods. + self._vars = {} + + # Following variables are for multipart templates. + self._current_part = 1 + self._current_pos = 0 + + def set(self, var, value): + """ Associate a value with top-level template variable or loop. + + A template identifier can represent either an ordinary variable + (string) or a loop. + + To assign a value to a string identifier pass a scalar + as the 'value' parameter. This scalar will be automatically + converted to string. + + To assign a value to a loop identifier pass a list of mappings as + the 'value' parameter. The engine iterates over this list and + assigns values from the mappings to variables in a template loop + block if a key in the mapping corresponds to a name of a variable + in the loop block. The number of mappings contained in this list + is equal to number of times the loop block is repeated in the + output. + + @header set(var, value) + @return No return value. + + @param var Name of template variable or loop. + @param value The value to associate. + + """ + # The correctness of character case is verified only for top-level + # variables. + if self.is_ordinary_var(value): + # template top-level ordinary variable + if not var.islower(): + raise TemplateError, "Invalid variable name '%s'." % var + elif type(value) == ListType: + # template top-level loop + if var != var.capitalize(): + raise TemplateError, "Invalid loop name '%s'." % var + else: + raise TemplateError, "Value of toplevel variable '%s' must "\ + "be either a scalar or a list." % var + self._vars[var] = value + self.DEB("VALUE SET: " + str(var)) + + def reset(self, keep_data=0): + """ Reset the template data. + + This method resets the data contained in the template processor + instance. The template processor instance can be used to process + any number of templates, but this method must be called after + a template is processed to reuse the instance, + + @header reset(keep_data=0) + @return No return value. + + @param keep_data Do not reset the template data. + Use this flag if you do not want the template data to be erased. + This way you can reuse the data contained in the instance of + the TemplateProcessor. + """ + self._current_part = 1 + self._current_pos = 0 + if not keep_data: + self._vars.clear() + self.DEB("RESET") + + def process(self, template, part=None): + """ Process a compiled template. Return the result as string. + + This method actually processes a template and returns + the result. + + @header process(template, part=None) + @return Result of the processing as string. + + @param template A compiled template. + Value of this parameter must be an instance of the + Template class created either by the + TemplateManager or by the TemplateCompiler. + + @param part The part of a multipart template to process. + This parameter can be used only together with a multipart + template. It specifies the number of the part to process. + It must be greater than zero, because the parts are numbered + from one. + + The parts must be processed in the right order. You + cannot process a part which precedes an already processed part. + + If this parameter is not specified, then the whole template + is processed, or all remaining parts are processed. + """ + self.DEB("APP INPUT:") + if self._debug: pprint.pprint(self._vars, sys.stderr) + if part != None and (part == 0 or part < self._current_part): + raise TemplateError, "process() - invalid part number" + + # This flag means "jump behind the end of current statement" or + # "skip the parameters of current statement". + # Even parameters that actually are not present in the template + # do appear in the list of tokens as empty items ! + skip_params = 0 + + # Stack for enabling or disabling output in response to TMPL_IF, + # TMPL_UNLESS, TMPL_ELSE and TMPL_LOOPs with no passes. + output_control = [] + ENABLE_OUTPUT = 1 + DISABLE_OUTPUT = 0 + + # Stacks for data related to loops. + loop_name = [] # name of a loop + loop_pass = [] # current pass of a loop (counted from zero) + loop_start = [] # index of loop start in token list + loop_total = [] # total number of passes in a loop + + tokens = template.tokens() + len_tokens = len(tokens) + out = "" # buffer for processed output + + # Recover position at which we ended after processing of last part. + i = self._current_pos + + # Process the list of tokens. + while 1: + if i == len_tokens: break + if skip_params: + # Skip the parameters following a statement. + skip_params = 0 + i += PARAMS_NUMBER + continue + + token = tokens[i] + if token.startswith("."
+ escape = tokens[i + PARAM_ESCAPE]
+ globalp = tokens[i + PARAM_GLOBAL]
+ skip_params = 1
+
+ # If output of current block is not disabled then append
+ # the substitued and escaped variable to the output.
+ if DISABLE_OUTPUT not in output_control:
+ value = str(self.find_value(var, loop_name, loop_pass,
+ loop_total, globalp))
+ out += self.escape(value, escape)
+ self.DEB("VAR: " + str(var))
+
+ elif token == "."
+ skip_params = 1
+
+ # Find total number of passes in this loop.
+ passtotal = self.find_value(var, loop_name, loop_pass,
+ loop_total)
+ if not passtotal: passtotal = 0
+ # Push data for this loop on the stack.
+ loop_total.append(passtotal)
+ loop_start.append(i)
+ loop_pass.append(0)
+ loop_name.append(var)
+
+ # Disable output of loop block if the number of passes
+ # in this loop is zero.
+ if passtotal == 0:
+ # This loop is empty.
+ output_control.append(DISABLE_OUTPUT)
+ self.DEB("LOOP: DISABLE: " + str(var))
+ else:
+ output_control.append(ENABLE_OUTPUT)
+ self.DEB("LOOP: FIRST PASS: %s TOTAL: %d"\
+ % (var, passtotal))
+
+ elif token == "."
+ globalp = tokens[i + PARAM_GLOBAL]
+ skip_params = 1
+ if self.find_value(var, loop_name, loop_pass,
+ loop_total, globalp):
+ output_control.append(ENABLE_OUTPUT)
+ self.DEB("IF: ENABLE: " + str(var))
+ else:
+ output_control.append(DISABLE_OUTPUT)
+ self.DEB("IF: DISABLE: " + str(var))
+
+ elif token == "."
+ globalp = tokens[i + PARAM_GLOBAL]
+ skip_params = 1
+ if self.find_value(var, loop_name, loop_pass,
+ loop_total, globalp):
+ output_control.append(DISABLE_OUTPUT)
+ self.DEB("UNLESS: DISABLE: " + str(var))
+ else:
+ output_control.append(ENABLE_OUTPUT)
+ self.DEB("UNLESS: ENABLE: " + str(var))
+
+ elif token == " ."
+
+ # If this loop was not disabled, then record the pass.
+ if loop_total[-1] > 0: loop_pass[-1] += 1
+
+ if loop_pass[-1] == loop_total[-1]:
+ # There are no more passes in this loop. Pop
+ # the loop from stack.
+ loop_pass.pop()
+ loop_name.pop()
+ loop_start.pop()
+ loop_total.pop()
+ output_control.pop()
+ self.DEB("LOOP: END")
+ else:
+ # Jump to the beggining of this loop block
+ # to process next pass of the loop.
+ i = loop_start[-1]
+ self.DEB("LOOP: NEXT PASS")
+
+ elif token == "."
+ output_control.pop()
+ self.DEB("IF: END")
+
+ elif token == "."
+ output_control.pop()
+ self.DEB("UNLESS: END")
+
+ elif token == "."
+ if output_control[-1] == DISABLE_OUTPUT:
+ # Condition was false, activate the ELSE block.
+ output_control[-1] = ENABLE_OUTPUT
+ self.DEB("ELSE: ENABLE")
+ elif output_control[-1] == ENABLE_OUTPUT:
+ # Condition was true, deactivate the ELSE block.
+ output_control[-1] = DISABLE_OUTPUT
+ self.DEB("ELSE: DISABLE")
+ else:
+ raise TemplateError, "BUG: ELSE: INVALID FLAG"
+
+ elif token == "
+ """ % filename + self.DEB("CANNOT INCLUDE WARNING") + + elif token == "." % token
+
+ elif DISABLE_OUTPUT not in output_control:
+ # Raw textual template data.
+ # If output of current block is not disabled, then
+ # append template data to the output buffer.
+ out += token
+
+ i += 1
+ # end of the big while loop
+
+ # Check whether all opening statements were closed.
+ if loop_name: raise TemplateError, "Missing ."
+ if output_control: raise TemplateError, "Missing or "
+ return out
+
+ ##############################################
+ # PRIVATE METHODS #
+ ##############################################
+
+ def DEB(self, str):
+ """ Print debugging message to stderr if debugging is enabled.
+ @hidden
+ """
+ if self._debug: print >> sys.stderr, str
+
+ def find_value(self, var, loop_name, loop_pass, loop_total,
+ global_override=None):
+ """ Search the self._vars data structure to find variable var
+ located in currently processed pass of a loop which
+ is currently being processed. If the variable is an ordinary
+ variable, then return it.
+
+ If the variable is an identificator of a loop, then
+ return the total number of times this loop will
+ be executed.
+
+ Return an empty string, if the variable is not
+ found at all.
+
+ @hidden
+ """
+ # Search for the requested variable in magic vars if the name
+ # of the variable starts with "__" and if we are inside a loop.
+ if self._magic_vars and var.startswith("__") and loop_name:
+ return self.magic_var(var, loop_pass[-1], loop_total[-1])
+
+ # Search for an ordinary variable or for a loop.
+ # Recursively search in self._vars for the requested variable.
+ scope = self._vars
+ globals = []
+ for i in range(len(loop_name)):
+ # If global lookup is on then push the value on the stack.
+ if ((self._global_vars and global_override != "0") or \
+ global_override == "1") and scope.has_key(var) and \
+ self.is_ordinary_var(scope[var]):
+ globals.append(scope[var])
+
+ # Descent deeper into the hierarchy.
+ if scope.has_key(loop_name[i]) and scope[loop_name[i]]:
+ scope = scope[loop_name[i]][loop_pass[i]]
+ else:
+ return ""
+
+ if scope.has_key(var):
+ # Value exists in current loop.
+ if type(scope[var]) == ListType:
+ # The requested value is a loop.
+ # Return total number of its passes.
+ return len(scope[var])
+ else:
+ return scope[var]
+ elif globals and \
+ ((self._global_vars and global_override != "0") or \
+ global_override == "1"):
+ # Return globally looked up value.
+ return globals.pop()
+ else:
+ # No value found.
+ if var[0].isupper():
+ # This is a loop name.
+ # Return zero, because the user wants to know number
+ # of its passes.
+ return 0
+ else:
+ return ""
+
+ def magic_var(self, var, loop_pass, loop_total):
+ """ Resolve and return value of a magic variable.
+ Raise an exception if the magic variable is not recognized.
+
+ @hidden
+ """
+ self.DEB("MAGIC: '%s', PASS: %d, TOTAL: %d"\
+ % (var, loop_pass, loop_total))
+ if var == "__FIRST__":
+ if loop_pass == 0:
+ return 1
+ else:
+ return 0
+ elif var == "__LAST__":
+ if loop_pass == loop_total - 1:
+ return 1
+ else:
+ return 0
+ elif var == "__INNER__":
+ # If this is neither the first nor the last pass.
+ if loop_pass != 0 and loop_pass != loop_total - 1:
+ return 1
+ else:
+ return 0
+ elif var == "__PASS__":
+ # Magic variable __PASS__ counts passes from one.
+ return loop_pass + 1
+ elif var == "__PASSTOTAL__":
+ return loop_total
+ elif var == "__ODD__":
+ # Internally pass numbers stored in loop_pass are counted from
+ # zero. But the template language presents them counted from one.
+ # Therefore we must add one to the actual loop_pass value to get
+ # the value we present to the user.
+ if (loop_pass + 1) % 2 != 0:
+ return 1
+ else:
+ return 0
+ elif var.startswith("__EVERY__"):
+ # Magic variable __EVERY__x is never true in first or last pass.
+ if loop_pass != 0 and loop_pass != loop_total - 1:
+ # Check if an integer follows the variable name.
+ try:
+ every = int(var[9:]) # nine is length of "__EVERY__"
+ except ValueError:
+ raise TemplateError, "Magic variable __EVERY__x: "\
+ "Invalid pass number."
+ else:
+ if not every:
+ raise TemplateError, "Magic variable __EVERY__x: "\
+ "Pass number cannot be zero."
+ elif (loop_pass + 1) % every == 0:
+ self.DEB("MAGIC: EVERY: " + str(every))
+ return 1
+ else:
+ return 0
+ else:
+ return 0
+ else:
+ raise TemplateError, "Invalid magic variable '%s'." % var
+
+ def escape(self, str, override=""):
+ """ Escape a string either by HTML escaping or by URL escaping.
+ @hidden
+ """
+ ESCAPE_QUOTES = 1
+ if (self._html_escape and override != "NONE" and override != "0" and \
+ override != "URL") or override == "HTML" or override == "1":
+ return cgi.escape(str, ESCAPE_QUOTES)
+ elif override == "URL":
+ return urllib.quote_plus(str)
+ else:
+ return str
+
+ def is_ordinary_var(self, var):
+ """ Return true if var is a scalar. (not a reference to loop)
+ @hidden
+ """
+ if type(var) == StringType or type(var) == IntType or \
+ type(var) == LongType or type(var) == FloatType:
+ return 1
+ else:
+ return 0
+
+
+##############################################
+# CLASS: TemplateCompiler #
+##############################################
+
+class TemplateCompiler:
+ """ Preprocess, parse, tokenize and compile the template.
+
+ This class parses the template and produces a 'compiled' form
+ of it. This compiled form is an instance of the Template
+ class. The compiled form is used as input for the TemplateProcessor
+ which uses it to actually process the template.
+
+ This class should be used direcly only when you need to compile
+ a template from a string. If your template is in a file, then you
+ should use the TemplateManager class which provides
+ a higher level interface to this class and also can save the
+ compiled template to disk in a precompiled form.
+ """
+
+ def __init__(self, include=1, max_include=5, comments=1, gettext=0,
+ debug=0):
+ """ Constructor.
+
+ @header __init__(include=1, max_include=5, comments=1, gettext=0,
+ debug=0)
+
+ @param include Enable or disable included templates.
+ @param max_include Maximum depth of nested inclusions.
+ @param comments Enable or disable template comments.
+ @param gettext Enable or disable gettext support.
+ @param debug Enable or disable debugging messages.
+ """
+
+ self._include = include
+ self._max_include = max_include
+ self._comments = comments
+ self._gettext = gettext
+ self._debug = debug
+
+ # This is a list of filenames of all included templates.
+ # It's modified by the include_templates() method.
+ self._include_files = []
+
+ # This is a counter of current inclusion depth. It's used to prevent
+ # infinite recursive includes.
+ self._include_level = 0
+
+ def compile(self, file):
+ """ Compile template from a file.
+
+ @header compile(file)
+ @return Compiled template.
+ The return value is an instance of the Template
+ class.
+
+ @param file Filename of the template.
+ See the prepare() method of the TemplateManager
+ class for exaplanation of this parameter.
+ """
+
+ self.DEB("COMPILING FROM FILE: " + file)
+ self._include_path = os.path.join(os.path.dirname(file), INCLUDE_DIR)
+ tokens = self.parse(self.read(file))
+ compile_params = (self._include, self._max_include, self._comments,
+ self._gettext)
+ return Template(__version__, file, self._include_files,
+ tokens, compile_params, self._debug)
+
+ def compile_string(self, data):
+ """ Compile template from a string.
+
+ This method compiles a template from a string. The
+ template cannot include any templates.
+ TMPL_INCLUDE statements are turned into warnings.
+
+ @header compile_string(data)
+ @return Compiled template.
+ The return value is an instance of the Template
+ class.
+
+ @param data String containing the template data.
+ """
+ self.DEB("COMPILING FROM STRING")
+ self._include = 0
+ tokens = self.parse(data)
+ compile_params = (self._include, self._max_include, self._comments,
+ self._gettext)
+ return Template(__version__, None, None, tokens, compile_params,
+ self._debug)
+
+ ##############################################
+ # PRIVATE METHODS #
+ ##############################################
+
+ def DEB(self, str):
+ """ Print debugging message to stderr if debugging is enabled.
+ @hidden
+ """
+ if self._debug: print >> sys.stderr, str
+
+ def read(self, filename):
+ """ Read content of file and return it. Raise an error if a problem
+ occurs.
+ @hidden
+ """
+ self.DEB("READING: " + filename)
+ try:
+ f = None
+ try:
+ f = open(filename, "r")
+ data = f.read()
+ except IOError, (errno, errstr):
+ raise TemplateError, "IO error while reading template '%s': "\
+ "(%d) %s" % (filename, errno, errstr)
+ else:
+ return data
+ finally:
+ if f: f.close()
+
+ def parse(self, template_data):
+ """ Parse the template. This method is recursively called from
+ within the include_templates() method.
+
+ @return List of processing tokens.
+ @hidden
+ """
+ if self._comments:
+ self.DEB("PREPROCESS: COMMENTS")
+ template_data = self.remove_comments(template_data)
+ tokens = self.tokenize(template_data)
+ if self._include:
+ self.DEB("PREPROCESS: INCLUDES")
+ self.include_templates(tokens)
+ return tokens
+
+ def remove_comments(self, template_data):
+ """ Remove comments from the template data.
+ @hidden
+ """
+ pattern = r"### .*"
+ return re.sub(pattern, "", template_data)
+
+ def include_templates(self, tokens):
+ """ Process TMPL_INCLUDE statements. Use the include_level counter
+ to prevent infinite recursion. Record paths to all included
+ templates to self._include_files.
+ @hidden
+ """
+ i = 0
+ out = "" # buffer for output
+ skip_params = 0
+
+ # Process the list of tokens.
+ while 1:
+ if i == len(tokens): break
+ if skip_params:
+ skip_params = 0
+ i += PARAMS_NUMBER
+ continue
+
+ token = tokens[i]
+ if token == "."
+ self._include_level += 1
+ if self._include_level > self._max_include:
+ # Do not include the template.
+ # Protection against infinite recursive includes.
+ skip_params = 1
+ self.DEB("INCLUDE: LIMIT REACHED: " + filename)
+ else:
+ # Include the template.
+ skip_params = 0
+ include_file = os.path.join(self._include_path, filename)
+ self._include_files.append(include_file)
+ include_data = self.read(include_file)
+ include_tokens = self.parse(include_data)
+
+ # Append the tokens from the included template to actual
+ # position in the tokens list, replacing the TMPL_INCLUDE
+ # token and its parameters.
+ tokens[i:i+PARAMS_NUMBER+1] = include_tokens
+ i = i + len(include_tokens)
+ self.DEB("INCLUDED: " + filename)
+ continue # Do not increment 'i' below.
+ i += 1
+ # end of the main while loop
+
+ if self._include_level > 0: self._include_level -= 1
+ return out
+
+ def tokenize(self, template_data):
+ """ Split the template into tokens separated by template statements.
+ The statements itself and associated parameters are also
+ separately included in the resulting list of tokens.
+ Return list of the tokens.
+
+ @hidden
+ """
+ self.DEB("TOKENIZING TEMPLATE")
+ # NOTE: The TWO double quotes in character class in the regexp below
+ # are there only to prevent confusion of syntax highlighter in Emacs.
+ pattern = r"""
+ (?:^[ \t]+)? # eat spaces, tabs (opt.)
+ (<
+ (?:!--[ ])? # comment start + space (opt.)
+ /?TMPL_[A-Z]+ # closing slash / (opt.) + statement
+ [ a-zA-Z0-9""/.=:_\\-]* # this spans also comments ending (--)
+ >)
+ [%s]? # eat trailing newline (opt.)
+ """ % os.linesep
+ rc = re.compile(pattern, re.VERBOSE | re.MULTILINE)
+ split = rc.split(template_data)
+ tokens = []
+ for statement in split:
+ if statement.startswith(" 0 and '=' not in params[0]:
+ # implicit identifier
+ name = params[0]
+ del params[0]
+ else:
+ # explicit identifier as a 'NAME' parameter
+ name = self.find_param("NAME", params)
+ self.DEB("TOKENIZER: NAME: " + str(name))
+ return name
+
+ def find_param(self, param, params):
+ """ Extract value of parameter from a statement.
+ @hidden
+ """
+ for pair in params:
+ name, value = pair.split("=")
+ if not name or not value:
+ raise TemplateError, "Syntax error in template."
+ if name == param:
+ if value[0] == '"':
+ # The value is in double quotes.
+ ret_value = value[1:-1]
+ else:
+ # The value is without double quotes.
+ ret_value = value
+ self.DEB("TOKENIZER: PARAM: '%s' => '%s'" % (param, ret_value))
+ return ret_value
+ else:
+ self.DEB("TOKENIZER: PARAM: '%s' => NOT DEFINED" % param)
+ return None
+
+
+##############################################
+# CLASS: Template #
+##############################################
+
+class Template:
+ """ This class represents a compiled template.
+
+ This class provides storage and methods for the compiled template
+ and associated metadata. It's serialized by pickle if we need to
+ save the compiled template to disk in a precompiled form.
+
+ You should never instantiate this class directly. Always use the
+ TemplateManager or TemplateCompiler classes to
+ create the instances of this class.
+
+ The only method which you can directly use is the is_uptodate
+ method.
+ """
+
+ def __init__(self, version, file, include_files, tokens, compile_params,
+ debug=0):
+ """ Constructor.
+ @hidden
+ """
+ self._version = version
+ self._file = file
+ self._tokens = tokens
+ self._compile_params = compile_params
+ self._debug = debug
+ self._mtime = None
+ self._include_mtimes = {}
+
+ if not file:
+ self.DEB("TEMPLATE WAS COMPILED FROM A STRING")
+ return
+
+ # Save modifitcation time of the main template file.
+ if os.path.isfile(file):
+ self._mtime = os.path.getmtime(file)
+ else:
+ raise TemplateError, "Template: file does not exist: '%s'" % file
+
+ # Save modificaton times of all included template files.
+ for inc_file in include_files:
+ if os.path.isfile(inc_file):
+ self._include_mtimes[inc_file] = os.path.getmtime(inc_file)
+ else:
+ raise TemplateError, "Template: file does not exist: '%s'"\
+ % inc_file
+
+ self.DEB("NEW TEMPLATE CREATED")
+
+ def is_uptodate(self, compile_params=None):
+ """ Check whether the compiled template is uptodate.
+
+ Return true if this compiled template is uptodate.
+ Return false, if the template source file was changed on the
+ disk since it was compiled.
+ Works by comparison of modification times.
+ Also takes modification times of all included templates
+ into account.
+
+ @header is_uptodate(compile_params=None)
+ @return True if the template is uptodate, false otherwise.
+
+ @param compile_params Only for internal use.
+ Do not use this optional parameter. It's intended only for
+ internal use by the TemplateManager.
+ """
+ if not self._file:
+ self.DEB("TEMPLATE COMPILED FROM A STRING")
+ return 0
+
+ if self._version != __version__:
+ self.DEB("TEMPLATE: VERSION NOT UPTODATE")
+ return 0
+
+ if compile_params != None and compile_params != self._compile_params:
+ self.DEB("TEMPLATE: DIFFERENT COMPILATION PARAMS")
+ return 0
+
+ # Check modification times of the main template and all included
+ # templates. If the included template no longer exists, then
+ # the problem will be resolved when the template is recompiled.
+
+ # Main template file.
+ if not (os.path.isfile(self._file) and \
+ self._mtime == os.path.getmtime(self._file)):
+ self.DEB("TEMPLATE: NOT UPTODATE: " + self._file)
+ return 0
+
+ # Included templates.
+ for inc_file in self._include_mtimes.keys():
+ if not (os.path.isfile(inc_file) and \
+ self._include_mtimes[inc_file] == \
+ os.path.getmtime(inc_file)):
+ self.DEB("TEMPLATE: NOT UPTODATE: " + inc_file)
+ return 0
+ else:
+ self.DEB("TEMPLATE: UPTODATE")
+ return 1
+
+ def tokens(self):
+ """ Get tokens of this template.
+ @hidden
+ """
+ return self._tokens
+
+ def file(self):
+ """ Get filename of the main file of this template.
+ @hidden
+ """
+ return self._file
+
+ def debug(self, debug):
+ """ Get debugging state.
+ @hidden
+ """
+ self._debug = debug
+
+ ##############################################
+ # PRIVATE METHODS #
+ ##############################################
+
+ def __getstate__(self):
+ """ Used by pickle when the class is serialized.
+ Remove the 'debug' attribute before serialization.
+ @hidden
+ """
+ dict = copy.copy(self.__dict__)
+ del dict["_debug"]
+ return dict
+
+ def __setstate__(self, dict):
+ """ Used by pickle when the class is unserialized.
+ Add the 'debug' attribute.
+ @hidden
+ """
+ dict["_debug"] = 0
+ self.__dict__ = dict
+
+
+ def DEB(self, str):
+ """ Print debugging message to stderr.
+ @hidden
+ """
+ if self._debug: print >> sys.stderr, str
+
+
+##############################################
+# EXCEPTIONS #
+##############################################
+
+class TemplateError(Exception):
+ """ Fatal exception. Raised on runtime or template syntax errors.
+
+ This exception is raised when a runtime error occurs or when a syntax
+ error in the template is found. It has one parameter which always
+ is a string containing a description of the error.
+
+ All potential IOError exceptions are handled by the module and are
+ converted to TemplateError exceptions. That means you should catch the
+ TemplateError exception if there is a possibility that for example
+ the template file will not be accesssible.
+
+ The exception can be raised by constructors or by any method of any
+ class.
+
+ The instance is no longer usable when this exception is raised.
+ """
+
+ def __init__(self, error):
+ """ Constructor.
+ @hidden
+ """
+ Exception.__init__(self, "Htmltmpl error: " + error)
+
+
+class PrecompiledError(Exception):
+ """ This exception is _PRIVATE_ and non fatal.
+ @hidden
+ """
+
+ def __init__(self, template):
+ """ Constructor.
+ @hidden
+ """
+ Exception.__init__(self, template)
+
diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/sanitize.py b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/sanitize.py
new file mode 100755
index 0000000..c98b14d
--- /dev/null
+++ b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/sanitize.py
@@ -0,0 +1,354 @@
+"""
+sanitize: bringing sanitiy to world of messed-up data
+"""
+
+__author__ = ["Mark Pilgrim
+# or utidylib '):
+ data = data.split('>', 1)[1]
+ if data.count(' ' % self.url)
+
+ def test_changedurl(self):
+ # change the URL directly
+ self.channel.url = self.changed_url
+ self.assertEqual(self.channel.feed_information(),
+ "<%s> (formerly <%s>)" % (self.changed_url, self.url))
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/tests/test_main.py b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/tests/test_main.py
new file mode 100755
index 0000000..c2be62d
--- /dev/null
+++ b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/tests/test_main.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+import os, sys, shutil, errno, unittest
+from ConfigParser import ConfigParser
+from StringIO import StringIO
+import planet
+
+class MainTest(unittest.TestCase):
+
+ def test_minimal(self):
+ configp = ConfigParser()
+ my_planet = planet.Planet(configp)
+ my_planet.run("Planet Name", "http://example.com", [])
+
+ def test_onefeed(self):
+ configp = ConfigParser()
+ configp.readfp(StringIO("""[http://www.example.com/]
+name = Mary
+"""))
+ my_planet = planet.Planet(configp)
+ my_planet.run("Planet Name", "http://example.com", [], True)
+
+
+ def test_generateall(self):
+ configp = ConfigParser()
+ configp.readfp(StringIO("""[http://www.example.com/]
+name = Mary
+"""))
+ my_planet = planet.Planet(configp)
+ my_planet.run("Planet Name", "http://example.com", [], True)
+ basedir = os.path.join(os.path.dirname(os.path.abspath(sys.modules[__name__].__file__)), 'data')
+ os.mkdir(self.output_dir)
+ t_file_names = ['simple', 'simple2']
+ self._remove_cached_templates(basedir, t_file_names)
+ t_files = [os.path.join(basedir, t_file) + '.tmpl' for t_file in t_file_names]
+ my_planet.generate_all_files(t_files, "Planet Name",
+ 'http://example.com/', 'http://example.com/feed/', 'Mary', 'mary@example.com')
+ for file_name in t_file_names:
+ name = os.path.join(self.output_dir, file_name)
+ content = file(name).read()
+ self.assertEqual(content, 'Mary\n')
+
+ def _remove_cached_templates(self, basedir, template_files):
+ """
+ Remove the .tmplc files and force them to be rebuilt.
+
+ This is required mainly so that the tests don't fail in mysterious ways in
+ directories that have been moved, eg 'branches/my-branch' to
+ 'branches/mysterious-branch' -- the .tmplc files seem to remember their full
+ path
+ """
+ for file in template_files:
+ path = os.path.join(basedir, file + '.tmplc')
+ try:
+ os.remove(path)
+ except OSError, e:
+ # we don't care about the file not being there, we care about
+ # everything else
+ if e.errno != errno.ENOENT:
+ raise
+
+ def setUp(self):
+ super(MainTest, self).setUp()
+ self.output_dir = 'output'
+
+ def tearDown(self):
+ super(MainTest, self).tearDown()
+ shutil.rmtree(self.output_dir, ignore_errors = True)
+ shutil.rmtree('cache', ignore_errors = True)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/tests/test_sanitize.py b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/tests/test_sanitize.py
new file mode 100755
index 0000000..f0f1d42
--- /dev/null
+++ b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/tests/test_sanitize.py
@@ -0,0 +1,125 @@
+# adapted from http://www.iamcal.com/publish/articles/php/processing_html_part_2/
+# and from http://feedparser.org/tests/wellformed/sanitize/
+# by Aaron Swartz, 2006, public domain
+
+import unittest, new
+from planet import sanitize
+
+class SanitizeTest(unittest.TestCase): pass
+
+# each call to HTML adds a test case to SanitizeTest
+testcases = 0
+def HTML(a, b):
+ global testcases
+ testcases += 1
+ func = lambda self: self.assertEqual(sanitize.HTML(a), b)
+ method = new.instancemethod(func, None, SanitizeTest)
+ setattr(SanitizeTest, "test_%d" % testcases, method)
+
+## basics
+HTML("","")
+HTML("hello","hello")
+
+## balancing tags
+HTML("hello","hello")
+HTML("hello","hello")
+HTML("hello","hello")
+HTML("hello","hello")
+HTML("hello","hello")
+HTML("","")
+
+## trailing slashes
+HTML('![]()
','![]()
')
+HTML('![]()
','![]()
')
+HTML('','')
+
+## balancing angle brakets
+HTML('
','b>')
+HTML('','>')
+HTML('foofoo','b>foo')
+HTML('>')
+HTML('b><','b>')
+HTML('>','>')
+
+## attributes
+HTML('','')
+HTML('![]()
','![]()
')
+HTML('
','')
+
+## dangerous tags (a small sample)
+sHTML = lambda x: HTML(x, 'safe description')
+sHTML('safe description')
+sHTML('safe description')
+sHTML(' description')
+sHTML('safe
, tag='pre', attrs=[('class', 'screen')]
+ if _debug: sys.stderr.write('_BaseHTMLProcessor, unknown_starttag, tag=%s\n' % tag)
+ uattrs = []
+ # thanks to Kevin Marks for this breathtaking hack to deal with (valid) high-bit attribute values in UTF-8 feeds
+ for key, value in attrs:
+ if type(value) != type(u''):
+ value = unicode(value, self.encoding)
+ uattrs.append((unicode(key, self.encoding), value))
+ strattrs = u''.join([u' %s="%s"' % (key, value) for key, value in uattrs]).encode(self.encoding)
+ if tag in self.elements_no_end_tag:
+ self.pieces.append('<%(tag)s%(strattrs)s />' % locals())
+ else:
+ self.pieces.append('<%(tag)s%(strattrs)s>' % locals())
+
+ def unknown_endtag(self, tag):
+ # called for each end tag, e.g. for , tag will be 'pre'
+ # Reconstruct the original end tag.
+ if tag not in self.elements_no_end_tag:
+ self.pieces.append("" % locals())
+
+ def handle_charref(self, ref):
+ # called for each character reference, e.g. for ' ', ref will be '160'
+ # Reconstruct the original character reference.
+ self.pieces.append('&#%(ref)s;' % locals())
+
+ def handle_entityref(self, ref):
+ # called for each entity reference, e.g. for '©', ref will be 'copy'
+ # Reconstruct the original entity reference.
+ import htmlentitydefs
+ if not hasattr(htmlentitydefs, 'name2codepoint') or htmlentitydefs.name2codepoint.has_key(ref):
+ self.pieces.append('&%(ref)s;' % locals())
+ else:
+ self.pieces.append('&%(ref)s' % locals())
+
+ def handle_data(self, text):
+ # called for each block of plain text, i.e. outside of any tag and
+ # not containing any character or entity references
+ # Store the original text verbatim.
+ if _debug: sys.stderr.write('_BaseHTMLProcessor, handle_text, text=%s\n' % text)
+ self.pieces.append(text)
+
+ def handle_comment(self, text):
+ # called for each HTML comment, e.g.
+ # Reconstruct the original comment.
+ self.pieces.append('' % locals())
+
+ def handle_pi(self, text):
+ # called for each processing instruction, e.g.
+ # Reconstruct original processing instruction.
+ self.pieces.append('' % locals())
+
+ def handle_decl(self, text):
+ # called for the DOCTYPE, if present, e.g.
+ #
+ # Reconstruct original DOCTYPE
+ self.pieces.append('' % locals())
+
+ _new_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9:]*\s*').match
+ def _scan_name(self, i, declstartpos):
+ rawdata = self.rawdata
+ n = len(rawdata)
+ if i == n:
+ return None, -1
+ m = self._new_declname_match(rawdata, i)
+ if m:
+ s = m.group()
+ name = s.strip()
+ if (i + len(s)) == n:
+ return None, -1 # end of buffer
+ return name.lower(), m.end()
+ else:
+ self.handle_data(rawdata)
+# self.updatepos(declstartpos, i)
+ return None, -1
+
+ def output(self):
+ '''Return processed HTML as a single string'''
+ return ''.join([str(p) for p in self.pieces])
+
+class _LooseFeedParser(_FeedParserMixin, _BaseHTMLProcessor):
+ def __init__(self, baseuri, baselang, encoding):
+ sgmllib.SGMLParser.__init__(self)
+ _FeedParserMixin.__init__(self, baseuri, baselang, encoding)
+
+ def decodeEntities(self, element, data):
+ data = data.replace('<', '<')
+ data = data.replace('<', '<')
+ data = data.replace('<', '<')
+ data = data.replace('>', '>')
+ data = data.replace('>', '>')
+ data = data.replace('>', '>')
+ data = data.replace('&', '&')
+ data = data.replace('&', '&')
+ data = data.replace('"', '"')
+ data = data.replace('"', '"')
+ data = data.replace(''', ''')
+ data = data.replace(''', ''')
+ if self.contentparams.has_key('type') and not self.contentparams.get('type', 'xml').endswith('xml'):
+ data = data.replace('<', '<')
+ data = data.replace('>', '>')
+ data = data.replace('&', '&')
+ data = data.replace('"', '"')
+ data = data.replace(''', "'")
+ return data
+
+ def strattrs(self, attrs):
+ return ''.join([' %s="%s"' % t for t in attrs])
+
+class _RelativeURIResolver(_BaseHTMLProcessor):
+ relative_uris = [('a', 'href'),
+ ('applet', 'codebase'),
+ ('area', 'href'),
+ ('blockquote', 'cite'),
+ ('body', 'background'),
+ ('del', 'cite'),
+ ('form', 'action'),
+ ('frame', 'longdesc'),
+ ('frame', 'src'),
+ ('iframe', 'longdesc'),
+ ('iframe', 'src'),
+ ('head', 'profile'),
+ ('img', 'longdesc'),
+ ('img', 'src'),
+ ('img', 'usemap'),
+ ('input', 'src'),
+ ('input', 'usemap'),
+ ('ins', 'cite'),
+ ('link', 'href'),
+ ('object', 'classid'),
+ ('object', 'codebase'),
+ ('object', 'data'),
+ ('object', 'usemap'),
+ ('q', 'cite'),
+ ('script', 'src')]
+
+ def __init__(self, baseuri, encoding):
+ _BaseHTMLProcessor.__init__(self, encoding)
+ self.baseuri = baseuri
+
+ def resolveURI(self, uri):
+ return _urljoin(self.baseuri, uri)
+
+ def unknown_starttag(self, tag, attrs):
+ attrs = self.normalize_attrs(attrs)
+ attrs = [(key, ((tag, key) in self.relative_uris) and self.resolveURI(value) or value) for key, value in attrs]
+ _BaseHTMLProcessor.unknown_starttag(self, tag, attrs)
+
+def _resolveRelativeURIs(htmlSource, baseURI, encoding):
+ if _debug: sys.stderr.write('entering _resolveRelativeURIs\n')
+ p = _RelativeURIResolver(baseURI, encoding)
+ p.feed(htmlSource)
+ return p.output()
+
+class _HTMLSanitizer(_BaseHTMLProcessor):
+ acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'b', 'big',
+ 'blockquote', 'br', 'button', 'caption', 'center', 'cite', 'code', 'col',
+ 'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt', 'em', 'fieldset',
+ 'font', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'input',
+ 'ins', 'kbd', 'label', 'legend', 'li', 'map', 'menu', 'ol', 'optgroup',
+ 'option', 'p', 'pre', 'q', 's', 'samp', 'select', 'small', 'span', 'strike',
+ 'strong', 'sub', 'sup', 'table', 'tbody', 'td', 'textarea', 'tfoot', 'th',
+ 'thead', 'tr', 'tt', 'u', 'ul', 'var']
+
+ acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey',
+ 'action', 'align', 'alt', 'axis', 'border', 'cellpadding', 'cellspacing',
+ 'char', 'charoff', 'charset', 'checked', 'cite', 'class', 'clear', 'cols',
+ 'colspan', 'color', 'compact', 'coords', 'datetime', 'dir', 'disabled',
+ 'enctype', 'for', 'frame', 'headers', 'height', 'href', 'hreflang', 'hspace',
+ 'id', 'ismap', 'label', 'lang', 'longdesc', 'maxlength', 'media', 'method',
+ 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'prompt', 'readonly',
+ 'rel', 'rev', 'rows', 'rowspan', 'rules', 'scope', 'selected', 'shape', 'size',
+ 'span', 'src', 'start', 'summary', 'tabindex', 'target', 'title', 'type',
+ 'usemap', 'valign', 'value', 'vspace', 'width', 'xml:lang']
+
+ unacceptable_elements_with_end_tag = ['script', 'applet']
+
+ def reset(self):
+ _BaseHTMLProcessor.reset(self)
+ self.unacceptablestack = 0
+
+ def unknown_starttag(self, tag, attrs):
+ if not tag in self.acceptable_elements:
+ if tag in self.unacceptable_elements_with_end_tag:
+ self.unacceptablestack += 1
+ return
+ attrs = self.normalize_attrs(attrs)
+ attrs = [(key, value) for key, value in attrs if key in self.acceptable_attributes]
+ _BaseHTMLProcessor.unknown_starttag(self, tag, attrs)
+
+ def unknown_endtag(self, tag):
+ if not tag in self.acceptable_elements:
+ if tag in self.unacceptable_elements_with_end_tag:
+ self.unacceptablestack -= 1
+ return
+ _BaseHTMLProcessor.unknown_endtag(self, tag)
+
+ def handle_pi(self, text):
+ pass
+
+ def handle_decl(self, text):
+ pass
+
+ def handle_data(self, text):
+ if not self.unacceptablestack:
+ _BaseHTMLProcessor.handle_data(self, text)
+
+def _sanitizeHTML(htmlSource, encoding):
+ p = _HTMLSanitizer(encoding)
+ p.feed(htmlSource)
+ data = p.output()
+ if TIDY_MARKUP:
+ # loop through list of preferred Tidy interfaces looking for one that's installed,
+ # then set up a common _tidy function to wrap the interface-specific API.
+ _tidy = None
+ for tidy_interface in PREFERRED_TIDY_INTERFACES:
+ try:
+ if tidy_interface == "uTidy":
+ from tidy import parseString as _utidy
+ def _tidy(data, **kwargs):
+ return str(_utidy(data, **kwargs))
+ break
+ elif tidy_interface == "mxTidy":
+ from mx.Tidy import Tidy as _mxtidy
+ def _tidy(data, **kwargs):
+ nerrors, nwarnings, data, errordata = _mxtidy.tidy(data, **kwargs)
+ return data
+ break
+ except:
+ pass
+ if _tidy:
+ utf8 = type(data) == type(u'')
+ if utf8:
+ data = data.encode('utf-8')
+ data = _tidy(data, output_xhtml=1, numeric_entities=1, wrap=0, char_encoding="utf8")
+ if utf8:
+ data = unicode(data, 'utf-8')
+ if data.count('tags in +# encoded HTML (skadz); fixed unicode handling in normalize_attrs (ChrisL); +# fixed relative URI processing for guid (skadz); added ICBM support; added +# base64 support +#2.7.5 - 1/15/2004 - MAP - added workaround for malformed DOCTYPE (seen on many +# blogspot.com sites); added _debug variable +#2.7.6 - 1/16/2004 - MAP - fixed bug with StringIO importing +#3.0b3 - 1/23/2004 - MAP - parse entire feed with real XML parser (if available); +# added several new supported namespaces; fixed bug tracking naked markup in +# description; added support for enclosure; added support for source; re-added +# support for cloud which got dropped somehow; added support for expirationDate +#3.0b4 - 1/26/2004 - MAP - fixed xml:lang inheritance; fixed multiple bugs tracking +# xml:base URI, one for documents that don't define one explicitly and one for +# documents that define an outer and an inner xml:base that goes out of scope +# before the end of the document +#3.0b5 - 1/26/2004 - MAP - fixed bug parsing multiple links at feed level +#3.0b6 - 1/27/2004 - MAP - added feed type and version detection, result['version'] +# will be one of SUPPORTED_VERSIONS.keys() or empty string if unrecognized; +# added support for creativeCommons:license and cc:license; added support for +# full Atom content model in title, tagline, info, copyright, summary; fixed bug +# with gzip encoding (not always telling server we support it when we do) +#3.0b7 - 1/28/2004 - MAP - support Atom-style author element in author_detail +# (dictionary of 'name', 'url', 'email'); map author to author_detail if author +# contains name + email address +#3.0b8 - 1/28/2004 - MAP - added support for contributor +#3.0b9 - 1/29/2004 - MAP - fixed check for presence of dict function; added +# support for summary +#3.0b10 - 1/31/2004 - MAP - incorporated ISO-8601 date parsing routines from +# xml.util.iso8601 +#3.0b11 - 2/2/2004 - MAP - added 'rights' to list of elements that can contain +# dangerous markup; fiddled with decodeEntities (not right); liberalized +# date parsing even further +#3.0b12 - 2/6/2004 - MAP - fiddled with decodeEntities (still not right); +# added support to Atom 0.2 subtitle; added support for Atom content model +# in copyright; better sanitizing of dangerous HTML elements with end tags +# (script, frameset) +#3.0b13 - 2/8/2004 - MAP - better handling of empty HTML tags (br, hr, img, +# etc.) in embedded markup, in either HTML or XHTML form (
,
,
) +#3.0b14 - 2/8/2004 - MAP - fixed CDATA handling in non-wellformed feeds under +# Python 2.1 +#3.0b15 - 2/11/2004 - MAP - fixed bug resolving relative links in wfw:commentRSS; +# fixed bug capturing author and contributor URL; fixed bug resolving relative +# links in author and contributor URL; fixed bug resolvin relative links in +# generator URL; added support for recognizing RSS 1.0; passed Simon Fell's +# namespace tests, and included them permanently in the test suite with his +# permission; fixed namespace handling under Python 2.1 +#3.0b16 - 2/12/2004 - MAP - fixed support for RSS 0.90 (broken in b15) +#3.0b17 - 2/13/2004 - MAP - determine character encoding as per RFC 3023 +#3.0b18 - 2/17/2004 - MAP - always map description to summary_detail (Andrei); +# use libxml2 (if available) +#3.0b19 - 3/15/2004 - MAP - fixed bug exploding author information when author +# name was in parentheses; removed ultra-problematic mxTidy support; patch to +# workaround crash in PyXML/expat when encountering invalid entities +# (MarkMoraes); support for textinput/textInput +#3.0b20 - 4/7/2004 - MAP - added CDF support +#3.0b21 - 4/14/2004 - MAP - added Hot RSS support +#3.0b22 - 4/19/2004 - MAP - changed 'channel' to 'feed', 'item' to 'entries' in +# results dict; changed results dict to allow getting values with results.key +# as well as results[key]; work around embedded illformed HTML with half +# a DOCTYPE; work around malformed Content-Type header; if character encoding +# is wrong, try several common ones before falling back to regexes (if this +# works, bozo_exception is set to CharacterEncodingOverride); fixed character +# encoding issues in BaseHTMLProcessor by tracking encoding and converting +# from Unicode to raw strings before feeding data to sgmllib.SGMLParser; +# convert each value in results to Unicode (if possible), even if using +# regex-based parsing +#3.0b23 - 4/21/2004 - MAP - fixed UnicodeDecodeError for feeds that contain +# high-bit characters in attributes in embedded HTML in description (thanks +# Thijs van de Vossen); moved guid, date, and date_parsed to mapped keys in +# FeedParserDict; tweaked FeedParserDict.has_key to return True if asking +# about a mapped key +#3.0fc1 - 4/23/2004 - MAP - made results.entries[0].links[0] and +# results.entries[0].enclosures[0] into FeedParserDict; fixed typo that could +# cause the same encoding to be tried twice (even if it failed the first time); +# fixed DOCTYPE stripping when DOCTYPE contained entity declarations; +# better textinput and image tracking in illformed RSS 1.0 feeds +#3.0fc2 - 5/10/2004 - MAP - added and passed Sam's amp tests; added and passed +# my blink tag tests +#3.0fc3 - 6/18/2004 - MAP - fixed bug in _changeEncodingDeclaration that +# failed to parse utf-16 encoded feeds; made source into a FeedParserDict; +# duplicate admin:generatorAgent/@rdf:resource in generator_detail.url; +# added support for image; refactored parse() fallback logic to try other +# encodings if SAX parsing fails (previously it would only try other encodings +# if re-encoding failed); remove unichr madness in normalize_attrs now that +# we're properly tracking encoding in and out of BaseHTMLProcessor; set +# feed.language from root-level xml:lang; set entry.id from rdf:about; +# send Accept header +#3.0 - 6/21/2004 - MAP - don't try iso-8859-1 (can't distinguish between +# iso-8859-1 and windows-1252 anyway, and most incorrectly marked feeds are +# windows-1252); fixed regression that could cause the same encoding to be +# tried twice (even if it failed the first time) +#3.0.1 - 6/22/2004 - MAP - default to us-ascii for all text/* content types; +# recover from malformed content-type header parameter with no equals sign +# ('text/xml; charset:iso-8859-1') +#3.1 - 6/28/2004 - MAP - added and passed tests for converting HTML entities +# to Unicode equivalents in illformed feeds (aaronsw); added and +# passed tests for converting character entities to Unicode equivalents +# in illformed feeds (aaronsw); test for valid parsers when setting +# XML_AVAILABLE; make version and encoding available when server returns +# a 304; add handlers parameter to pass arbitrary urllib2 handlers (like +# digest auth or proxy support); add code to parse username/password +# out of url and send as basic authentication; expose downloading-related +# exceptions in bozo_exception (aaronsw); added __contains__ method to +# FeedParserDict (aaronsw); added publisher_detail (aaronsw) +#3.2 - 7/3/2004 - MAP - use cjkcodecs and iconv_codec if available; always +# convert feed to UTF-8 before passing to XML parser; completely revamped +# logic for determining character encoding and attempting XML parsing +# (much faster); increased default timeout to 20 seconds; test for presence +# of Location header on redirects; added tests for many alternate character +# encodings; support various EBCDIC encodings; support UTF-16BE and +# UTF16-LE with or without a BOM; support UTF-8 with a BOM; support +# UTF-32BE and UTF-32LE with or without a BOM; fixed crashing bug if no +# XML parsers are available; added support for 'Content-encoding: deflate'; +# send blank 'Accept-encoding: ' header if neither gzip nor zlib modules +# are available +#3.3 - 7/15/2004 - MAP - optimize EBCDIC to ASCII conversion; fix obscure +# problem tracking xml:base and xml:lang if element declares it, child +# doesn't, first grandchild redeclares it, and second grandchild doesn't; +# refactored date parsing; defined public registerDateHandler so callers +# can add support for additional date formats at runtime; added support +# for OnBlog, Nate, MSSQL, Greek, and Hungarian dates (ytrewq1); added +# zopeCompatibilityHack() which turns FeedParserDict into a regular +# dictionary, required for Zope compatibility, and also makes command- +# line debugging easier because pprint module formats real dictionaries +# better than dictionary-like objects; added NonXMLContentType exception, +# which is stored in bozo_exception when a feed is served with a non-XML +# media type such as 'text/plain'; respect Content-Language as default +# language if not xml:lang is present; cloud dict is now FeedParserDict; +# generator dict is now FeedParserDict; better tracking of xml:lang, +# including support for xml:lang='' to unset the current language; +# recognize RSS 1.0 feeds even when RSS 1.0 namespace is not the default +# namespace; don't overwrite final status on redirects (scenarios: +# redirecting to a URL that returns 304, redirecting to a URL that +# redirects to another URL with a different type of redirect); add +# support for HTTP 303 redirects +#4.0 - MAP - support for relative URIs in xml:base attribute; fixed +# encoding issue with mxTidy (phopkins); preliminary support for RFC 3229; +# support for Atom 1.0; support for iTunes extensions; new 'tags' for +# categories/keywords/etc. as array of dict +# {'term': term, 'scheme': scheme, 'label': label} to match Atom 1.0 +# terminology; parse RFC 822-style dates with no time; lots of other +# bug fixes +#4.1 - MAP - removed socket timeout; added support for chardet library diff --git a/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/htmltmpl.py b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/htmltmpl.py new file mode 100755 index 0000000..be6e41b --- /dev/null +++ b/DJAGEN/branches/mustafa_branch/djagen/gezegen/planet/htmltmpl.py @@ -0,0 +1,1480 @@ + +""" A templating engine for separation of code and HTML. + + The documentation of this templating engine is separated to two parts: + + 1. Description of the templating language. + + 2. Documentation of classes and API of this module that provides + a Python implementation of the templating language. + + All the documentation can be found in 'doc' directory of the + distribution tarball or at the homepage of the engine. + Latest versions of this module are also available at that website. + + You can use and redistribute this module under conditions of the + GNU General Public License that can be found either at + [ http://www.gnu.org/ ] or in file "LICENSE" contained in the + distribution tarball of this module. + + Copyright (c) 2001 Tomas Styblo, tripie@cpan.org + + @name htmltmpl + @version 1.22 + @author-name Tomas Styblo + @author-email tripie@cpan.org + @website http://htmltmpl.sourceforge.net/ + @license-name GNU GPL + @license-url http://www.gnu.org/licenses/gpl.html +""" + +__version__ = 1.22 +__author__ = "Tomas Styblo (tripie@cpan.org)" + +# All imported modules are part of the standard Python library. + +from types import * +import re +import os +import os.path +import pprint # only for debugging +import sys +import copy +import cgi # for HTML escaping of variables +import urllib # for URL escaping of variables +import cPickle # for template compilation +import gettext + +INCLUDE_DIR = "inc" + +# Total number of possible parameters. +# Increment if adding a parameter to any statement. +PARAMS_NUMBER = 3 + +# Relative positions of parameters in TemplateCompiler.tokenize(). +PARAM_NAME = 1 +PARAM_ESCAPE = 2 +PARAM_GLOBAL = 3 +PARAM_GETTEXT_STRING = 1 + +# Find a way to lock files. Currently implemented only for UNIX and windows. +LOCKTYPE_FCNTL = 1 +LOCKTYPE_MSVCRT = 2 +LOCKTYPE = None +try: + import fcntl +except: + try: + import msvcrt + except: + LOCKTYPE = None + else: + LOCKTYPE = LOCKTYPE_MSVCRT +else: + LOCKTYPE = LOCKTYPE_FCNTL +LOCK_EX = 1 +LOCK_SH = 2 +LOCK_UN = 3 + +############################################## +# CLASS: TemplateManager # +############################################## + +class TemplateManager: + """ Class that manages compilation and precompilation of templates. + + You should use this class whenever you work with templates + that are stored in a file. The class can create a compiled + template and transparently manage its precompilation. It also + keeps the precompiled templates up-to-date by modification times + comparisons. + """ + + def __init__(self, include=1, max_include=5, precompile=1, comments=1, + gettext=0, debug=0): + """ Constructor. + + @header + __init__(include=1, max_include=5, precompile=1, comments=1, + gettext=0, debug=0) + + @param include Enable or disable included templates. + This optional parameter can be used to enable or disable + TMPL_INCLUDE inclusion of templates. Disabling of + inclusion can improve performance a bit. The inclusion is + enabled by default. + + @param max_include Maximum depth of nested inclusions. + This optional parameter can be used to specify maximum depth of + nested TMPL_INCLUDE inclusions. It defaults to 5. + This setting prevents infinite recursive inclusions. + + @param precompile Enable or disable precompilation of templates. + This optional parameter can be used to enable or disable + creation and usage of precompiled templates. + + A precompiled template is saved to the same directory in + which the main template file is located. You need write + permissions to that directory. + + Precompilation provides a significant performance boost because + it's not necessary to parse the templates over and over again. + The boost is especially noticeable when templates that include + other templates are used. + + Comparison of modification times of the main template and all + included templates is used to ensure that the precompiled + templates are up-to-date. Templates are also recompiled if the + htmltmpl module is updated. + + The TemplateErrorexception is raised when the precompiled + template cannot be saved. Precompilation is enabled by default. + + Precompilation is available only on UNIX and Windows platforms, + because proper file locking which is necessary to ensure + multitask safe behaviour is platform specific and is not + implemented for other platforms. Attempts to enable precompilation + on the other platforms result in raise of the + TemplateError exception. + + @param comments Enable or disable template comments. + This optional parameter can be used to enable or disable + template comments. + Disabling of the comments can improve performance a bit. + Comments are enabled by default. + + @param gettext Enable or disable gettext support. + + @param debug Enable or disable debugging messages. + This optional parameter is a flag that can be used to enable + or disable debugging messages which are printed to the standard + error output. The debugging messages are disabled by default. + """ + # Save the optional parameters. + # These values are not modified by any method. + self._include = include + self._max_include = max_include + self._precompile = precompile + self._comments = comments + self._gettext = gettext + self._debug = debug + + # Find what module to use to lock files. + # File locking is necessary for the 'precompile' feature to be + # multitask/thread safe. Currently it works only on UNIX + # and Windows. Anyone willing to implement it on Mac ? + if precompile and not LOCKTYPE: + raise TemplateError, "Template precompilation is not "\ + "available on this platform." + self.DEB("INIT DONE") + + def prepare(self, file): + """ Preprocess, parse, tokenize and compile the template. + + If precompilation is enabled then this method tries to load + a precompiled form of the template from the same directory + in which the template source file is located. If it succeeds, + then it compares modification times stored in the precompiled + form to modification times of source files of the template, + including source files of all templates included via the + TMPL_INCLUDE statements. If any of the modification times + differs, then the template is recompiled and the precompiled + form updated. + + If precompilation is disabled, then this method parses and + compiles the template. + + @header prepare(file) + + @return Compiled template. + The methods returns an instance of the Template class + which is a compiled form of the template. This instance can be + used as input for the TemplateProcessor. + + @param file Path to the template file to prepare. + The method looks for the template file in current directory + if the parameter is a relative path. All included templates must + be placed in subdirectory 'inc' of the + directory in which the main template file is located. + """ + compiled = None + if self._precompile: + if self.is_precompiled(file): + try: + precompiled = self.load_precompiled(file) + except PrecompiledError, template: + print >> sys.stderr, "Htmltmpl: bad precompiled "\ + "template '%s' removed" % template + compiled = self.compile(file) + self.save_precompiled(compiled) + else: + precompiled.debug(self._debug) + compile_params = (self._include, self._max_include, + self._comments, self._gettext) + if precompiled.is_uptodate(compile_params): + self.DEB("PRECOMPILED: UPTODATE") + compiled = precompiled + else: + self.DEB("PRECOMPILED: NOT UPTODATE") + compiled = self.update(precompiled) + else: + self.DEB("PRECOMPILED: NOT PRECOMPILED") + compiled = self.compile(file) + self.save_precompiled(compiled) + else: + self.DEB("PRECOMPILATION DISABLED") + compiled = self.compile(file) + return compiled + + def update(self, template): + """ Update (recompile) a compiled template. + + This method recompiles a template compiled from a file. + If precompilation is enabled then the precompiled form saved on + disk is also updated. + + @header update(template) + + @return Recompiled template. + It's ensured that the returned template is up-to-date. + + @param template A compiled template. + This parameter should be an instance of the Template + class, created either by the TemplateManager or by the + TemplateCompiler. The instance must represent a template + compiled from a file on disk. + """ + self.DEB("UPDATE") + updated = self.compile(template.file()) + if self._precompile: + self.save_precompiled(updated) + return updated + + ############################################## + # PRIVATE METHODS # + ############################################## + + def DEB(self, str): + """ Print debugging message to stderr if debugging is enabled. + @hidden + """ + if self._debug: print >> sys.stderr, str + + def lock_file(self, file, lock): + """ Provide platform independent file locking. + @hidden + """ + fd = file.fileno() + if LOCKTYPE == LOCKTYPE_FCNTL: + if lock == LOCK_SH: + fcntl.flock(fd, fcntl.LOCK_SH) + elif lock == LOCK_EX: + fcntl.flock(fd, fcntl.LOCK_EX) + elif lock == LOCK_UN: + fcntl.flock(fd, fcntl.LOCK_UN) + else: + raise TemplateError, "BUG: bad lock in lock_file" + elif LOCKTYPE == LOCKTYPE_MSVCRT: + if lock == LOCK_SH: + # msvcrt does not support shared locks :-( + msvcrt.locking(fd, msvcrt.LK_LOCK, 1) + elif lock == LOCK_EX: + msvcrt.locking(fd, msvcrt.LK_LOCK, 1) + elif lock == LOCK_UN: + msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) + else: + raise TemplateError, "BUG: bad lock in lock_file" + else: + raise TemplateError, "BUG: bad locktype in lock_file" + + def compile(self, file): + """ Compile the template. + @hidden + """ + return TemplateCompiler(self._include, self._max_include, + self._comments, self._gettext, + self._debug).compile(file) + + def is_precompiled(self, file): + """ Return true if the template is already precompiled on the disk. + This method doesn't check whether the compiled template is + uptodate. + @hidden + """ + filename = file + "c" # "template.tmplc" + if os.path.isfile(filename): + return 1 + else: + return 0 + + def load_precompiled(self, file): + """ Load precompiled template from disk. + + Remove the precompiled template file and recompile it + if the file contains corrupted or unpicklable data. + + @hidden + """ + filename = file + "c" # "template.tmplc" + self.DEB("LOADING PRECOMPILED") + try: + remove_bad = 0 + file = None + try: + file = open(filename, "rb") + self.lock_file(file, LOCK_SH) + precompiled = cPickle.load(file) + except IOError, (errno, errstr): + raise TemplateError, "IO error in load precompiled "\ + "template '%s': (%d) %s"\ + % (filename, errno, errstr) + except cPickle.UnpicklingError: + remove_bad = 1 + raise PrecompiledError, filename + except: + remove_bad = 1 + raise + else: + return precompiled + finally: + if file: + self.lock_file(file, LOCK_UN) + file.close() + if remove_bad and os.path.isfile(filename): + # X: We may lose the original exception here, raising OSError. + os.remove(filename) + + def save_precompiled(self, template): + """ Save compiled template to disk in precompiled form. + + Associated metadata is also saved. It includes: filename of the + main template file, modification time of the main template file, + modification times of all included templates and version of the + htmltmpl module which compiled the template. + + The method removes a file which is saved only partially because + of some error. + + @hidden + """ + filename = template.file() + "c" # creates "template.tmplc" + # Check if we have write permission to the template's directory. + template_dir = os.path.dirname(os.path.abspath(filename)) + if not os.access(template_dir, os.W_OK): + raise TemplateError, "Cannot save precompiled templates "\ + "to '%s': write permission denied."\ + % template_dir + try: + remove_bad = 0 + file = None + try: + file = open(filename, "wb") # may truncate existing file + self.lock_file(file, LOCK_EX) + BINARY = 1 + READABLE = 0 + if self._debug: + cPickle.dump(template, file, READABLE) + else: + cPickle.dump(template, file, BINARY) + except IOError, (errno, errstr): + remove_bad = 1 + raise TemplateError, "IO error while saving precompiled "\ + "template '%s': (%d) %s"\ + % (filename, errno, errstr) + except cPickle.PicklingError, error: + remove_bad = 1 + raise TemplateError, "Pickling error while saving "\ + "precompiled template '%s': %s"\ + % (filename, error) + except: + remove_bad = 1 + raise + else: + self.DEB("SAVING PRECOMPILED") + finally: + if file: + self.lock_file(file, LOCK_UN) + file.close() + if remove_bad and os.path.isfile(filename): + # X: We may lose the original exception here, raising OSError. + os.remove(filename) + + +############################################## +# CLASS: TemplateProcessor # +############################################## + +class TemplateProcessor: + """ Fill the template with data and process it. + + This class provides actual processing of a compiled template. + Use it to set template variables and loops and then obtain + result of the processing. + """ + + def __init__(self, html_escape=1, magic_vars=1, global_vars=0, debug=0): + """ Constructor. + + @header __init__(html_escape=1, magic_vars=1, global_vars=0, + debug=0) + + @param html_escape Enable or disable HTML escaping of variables. + This optional parameter is a flag that can be used to enable or + disable automatic HTML escaping of variables. + All variables are by default automatically HTML escaped. + The escaping process substitutes HTML brackets, ampersands and + double quotes with appropriate HTML entities. + + @param magic_vars Enable or disable loop magic variables. + This parameter can be used to enable or disable + "magic" context variables, that are automatically defined inside + loops. Magic variables are enabled by default. + + Refer to the language specification for description of these + magic variables. + + @param global_vars Globally activate global lookup of variables. + This optional parameter is a flag that can be used to specify + whether variables which cannot be found in the current scope + should be automatically looked up in enclosing scopes. + + Automatic global lookup is disabled by default. Global lookup + can be overriden on a per-variable basis by the + GLOBAL parameter of a TMPL_VAR + statement. + + @param debug Enable or disable debugging messages. + """ + self._html_escape = html_escape + self._magic_vars = magic_vars + self._global_vars = global_vars + self._debug = debug + + # Data structure containing variables and loops set by the + # application. Use debug=1, process some template and + # then check stderr to see how the structure looks. + # It's modified only by set() and reset() methods. + self._vars = {} + + # Following variables are for multipart templates. + self._current_part = 1 + self._current_pos = 0 + + def set(self, var, value): + """ Associate a value with top-level template variable or loop. + + A template identifier can represent either an ordinary variable + (string) or a loop. + + To assign a value to a string identifier pass a scalar + as the 'value' parameter. This scalar will be automatically + converted to string. + + To assign a value to a loop identifier pass a list of mappings as + the 'value' parameter. The engine iterates over this list and + assigns values from the mappings to variables in a template loop + block if a key in the mapping corresponds to a name of a variable + in the loop block. The number of mappings contained in this list + is equal to number of times the loop block is repeated in the + output. + + @header set(var, value) + @return No return value. + + @param var Name of template variable or loop. + @param value The value to associate. + + """ + # The correctness of character case is verified only for top-level + # variables. + if self.is_ordinary_var(value): + # template top-level ordinary variable + if not var.islower(): + raise TemplateError, "Invalid variable name '%s'." % var + elif type(value) == ListType: + # template top-level loop + if var != var.capitalize(): + raise TemplateError, "Invalid loop name '%s'." % var + else: + raise TemplateError, "Value of toplevel variable '%s' must "\ + "be either a scalar or a list." % var + self._vars[var] = value + self.DEB("VALUE SET: " + str(var)) + + def reset(self, keep_data=0): + """ Reset the template data. + + This method resets the data contained in the template processor + instance. The template processor instance can be used to process + any number of templates, but this method must be called after + a template is processed to reuse the instance, + + @header reset(keep_data=0) + @return No return value. + + @param keep_data Do not reset the template data. + Use this flag if you do not want the template data to be erased. + This way you can reuse the data contained in the instance of + the TemplateProcessor. + """ + self._current_part = 1 + self._current_pos = 0 + if not keep_data: + self._vars.clear() + self.DEB("RESET") + + def process(self, template, part=None): + """ Process a compiled template. Return the result as string. + + This method actually processes a template and returns + the result. + + @header process(template, part=None) + @return Result of the processing as string. + + @param template A compiled template. + Value of this parameter must be an instance of the + Template class created either by the + TemplateManager or by the TemplateCompiler. + + @param part The part of a multipart template to process. + This parameter can be used only together with a multipart + template. It specifies the number of the part to process. + It must be greater than zero, because the parts are numbered + from one. + + The parts must be processed in the right order. You + cannot process a part which precedes an already processed part. + + If this parameter is not specified, then the whole template + is processed, or all remaining parts are processed. + """ + self.DEB("APP INPUT:") + if self._debug: pprint.pprint(self._vars, sys.stderr) + if part != None and (part == 0 or part < self._current_part): + raise TemplateError, "process() - invalid part number" + + # This flag means "jump behind the end of current statement" or + # "skip the parameters of current statement". + # Even parameters that actually are not present in the template + # do appear in the list of tokens as empty items ! + skip_params = 0 + + # Stack for enabling or disabling output in response to TMPL_IF, + # TMPL_UNLESS, TMPL_ELSE and TMPL_LOOPs with no passes. + output_control = [] + ENABLE_OUTPUT = 1 + DISABLE_OUTPUT = 0 + + # Stacks for data related to loops. + loop_name = [] # name of a loop + loop_pass = [] # current pass of a loop (counted from zero) + loop_start = [] # index of loop start in token list + loop_total = [] # total number of passes in a loop + + tokens = template.tokens() + len_tokens = len(tokens) + out = "" # buffer for processed output + + # Recover position at which we ended after processing of last part. + i = self._current_pos + + # Process the list of tokens. + while 1: + if i == len_tokens: break + if skip_params: + # Skip the parameters following a statement. + skip_params = 0 + i += PARAMS_NUMBER + continue + + token = tokens[i] + if token.startswith("
+ HTMLTMPL WARNING:
+ Cannot include template: %s
+
+ """ % filename + self.DEB("CANNOT INCLUDE WARNING") + + elif token == "
, tag='pre', attrs=[('class', 'screen')]
+ if _debug: sys.stderr.write('_BaseHTMLProcessor, unknown_starttag, tag=%s\n' % tag)
+ uattrs = []
+ # thanks to Kevin Marks for this breathtaking hack to deal with (valid) high-bit attribute values in UTF-8 feeds
+ for key, value in attrs:
+ if type(value) != type(u''):
+ value = unicode(value, self.encoding)
+ uattrs.append((unicode(key, self.encoding), value))
+ strattrs = u''.join([u' %s="%s"' % (key, value) for key, value in uattrs]).encode(self.encoding)
+ if tag in self.elements_no_end_tag:
+ self.pieces.append('<%(tag)s%(strattrs)s />' % locals())
+ else:
+ self.pieces.append('<%(tag)s%(strattrs)s>' % locals())
+
+ def unknown_endtag(self, tag):
+ # called for each end tag, e.g. for , tag will be 'pre'
+ # Reconstruct the original end tag.
+ if tag not in self.elements_no_end_tag:
+ self.pieces.append("" % locals())
+
+ def handle_charref(self, ref):
+ # called for each character reference, e.g. for ' ', ref will be '160'
+ # Reconstruct the original character reference.
+ self.pieces.append('&#%(ref)s;' % locals())
+
+ def handle_entityref(self, ref):
+ # called for each entity reference, e.g. for '©', ref will be 'copy'
+ # Reconstruct the original entity reference.
+ self.pieces.append('&%(ref)s;' % locals())
+
+ def handle_data(self, text):
+ # called for each block of plain text, i.e. outside of any tag and
+ # not containing any character or entity references
+ # Store the original text verbatim.
+ if _debug: sys.stderr.write('_BaseHTMLProcessor, handle_text, text=%s\n' % text)
+ self.pieces.append(text)
+
+ def handle_comment(self, text):
+ # called for each HTML comment, e.g.
+ # Reconstruct the original comment.
+ self.pieces.append('' % locals())
+
+ def handle_pi(self, text):
+ # called for each processing instruction, e.g.
+ # Reconstruct original processing instruction.
+ self.pieces.append('' % locals())
+
+ def handle_decl(self, text):
+ # called for the DOCTYPE, if present, e.g.
+ #
+ # Reconstruct original DOCTYPE
+ self.pieces.append('' % locals())
+
+ _new_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9:]*\s*').match
+ def _scan_name(self, i, declstartpos):
+ rawdata = self.rawdata
+ n = len(rawdata)
+ if i == n:
+ return None, -1
+ m = self._new_declname_match(rawdata, i)
+ if m:
+ s = m.group()
+ name = s.strip()
+ if (i + len(s)) == n:
+ return None, -1 # end of buffer
+ return name.lower(), m.end()
+ else:
+ self.handle_data(rawdata)
+# self.updatepos(declstartpos, i)
+ return None, -1
+
+ def output(self):
+ '''Return processed HTML as a single string'''
+ return ''.join([str(p) for p in self.pieces])
+
+class _HTMLSanitizer(_BaseHTMLProcessor):
+ acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'b', 'big',
+ 'blockquote', 'br', 'button', 'caption', 'center', 'cite', 'code', 'col',
+ 'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt', 'em', 'fieldset',
+ 'font', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'input',
+ 'ins', 'kbd', 'label', 'legend', 'li', 'map', 'menu', 'ol', 'optgroup',
+ 'option', 'p', 'pre', 'q', 's', 'samp', 'select', 'small', 'span', 'strike',
+ 'strong', 'sub', 'sup', 'table', 'textarea', 'tbody', 'td', 'tfoot', 'th',
+ 'thead', 'tr', 'tt', 'u', 'ul', 'var']
+
+ acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey',
+ 'action', 'align', 'alt', 'axis', 'border', 'cellpadding', 'cellspacing',
+ 'char', 'charoff', 'charset', 'checked', 'cite', 'class', 'clear', 'cols',
+ 'colspan', 'color', 'compact', 'coords', 'datetime', 'dir', 'disabled',
+ 'enctype', 'for', 'frame', 'headers', 'height', 'href', 'hreflang', 'hspace',
+ 'id', 'ismap', 'label', 'lang', 'longdesc', 'maxlength', 'media', 'method',
+ 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'prompt', 'readonly',
+ 'rel', 'rev', 'rows', 'rowspan', 'rules', 'scope', 'selected', 'shape', 'size',
+ 'span', 'src', 'start', 'summary', 'tabindex', 'target', 'title', 'type',
+ 'usemap', 'valign', 'value', 'vspace', 'width']
+
+ ignorable_elements = ['script', 'applet', 'style']
+
+ def reset(self):
+ _BaseHTMLProcessor.reset(self)
+ self.tag_stack = []
+ self.ignore_level = 0
+
+ def feed(self, data):
+ _BaseHTMLProcessor.feed(self, data)
+ while self.tag_stack:
+ _BaseHTMLProcessor.unknown_endtag(self, self.tag_stack.pop())
+
+ def unknown_starttag(self, tag, attrs):
+ if tag in self.ignorable_elements:
+ self.ignore_level += 1
+ return
+
+ if self.ignore_level:
+ return
+
+ if tag in self.acceptable_elements:
+ attrs = self.normalize_attrs(attrs)
+ attrs = [(key, value) for key, value in attrs if key in self.acceptable_attributes]
+ if tag not in self.elements_no_end_tag:
+ self.tag_stack.append(tag)
+ _BaseHTMLProcessor.unknown_starttag(self, tag, attrs)
+
+ def unknown_endtag(self, tag):
+ if tag in self.ignorable_elements:
+ self.ignore_level -= 1
+ return
+
+ if self.ignore_level:
+ return
+
+ if tag in self.acceptable_elements and tag not in self.elements_no_end_tag:
+ match = False
+ while self.tag_stack:
+ top = self.tag_stack.pop()
+ if top == tag:
+ match = True
+ break
+ _BaseHTMLProcessor.unknown_endtag(self, top)
+
+ if match:
+ _BaseHTMLProcessor.unknown_endtag(self, tag)
+
+ def handle_pi(self, text):
+ pass
+
+ def handle_decl(self, text):
+ pass
+
+ def handle_data(self, text):
+ if not self.ignore_level:
+ text = text.replace('<', '')
+ _BaseHTMLProcessor.handle_data(self, text)
+
+def HTML(htmlSource, encoding='utf8'):
+ p = _HTMLSanitizer(encoding)
+ p.feed(htmlSource)
+ data = p.output()
+ if TIDY_MARKUP:
+ # loop through list of preferred Tidy interfaces looking for one that's installed,
+ # then set up a common _tidy function to wrap the interface-specific API.
+ _tidy = None
+ for tidy_interface in PREFERRED_TIDY_INTERFACES:
+ try:
+ if tidy_interface == "uTidy":
+ from tidy import parseString as _utidy
+ def _tidy(data, **kwargs):
+ return str(_utidy(data, **kwargs))
+ break
+ elif tidy_interface == "mxTidy":
+ from mx.Tidy import Tidy as _mxtidy
+ def _tidy(data, **kwargs):
+ nerrors, nwarnings, data, errordata = _mxtidy.tidy(data, **kwargs)
+ return data
+ break
+ except:
+ pass
+ if _tidy:
+ utf8 = type(data) == type(u'')
+ if utf8:
+ data = data.encode('utf-8')
+ data = _tidy(data, output_xhtml=1, numeric_entities=1, wrap=0, char_encoding="utf8")
+ if utf8:
+ data = unicode(data, 'utf-8')
+ if data.count('