Integrate feed extractors, add more hard-coded feed URIs

3 files changed, 87 insertions, 7 deletions

diff --git a/extradata.py b/extradata.py
index b3bdc22..0d3867d 100644
--- a/extradata.py
+++ b/extradata.py
@@ -10,6 +10,9 @@ TRANSITION_DATA_EXTRA = {
 			'name':'Mark Lee',
 			'email':'malept@malept.com',
 		},
+		'feeds':[
+			'http://bazaar.launchpad.net/~serkan-kaba/serkan-overlay/mainline/atom',
+		],
 	},
 	'lila-theme':{
 		'maintainer':{
@@ -27,11 +30,22 @@ TRANSITION_DATA_EXTRA = {
 		'maintainer':{
 			'name':'Sebastian Pipping',
 		},
-		'feeds':[
-			'http://git.goodpoint.de/?p=overlay-sping.git;a=atom',
-			'http://git.goodpoint.de/?p=overlay-sping.git;a=rss',
-		],
 	},
+	'jyujin':{'feeds':['http://git.jyujin.de/?p=portage-overlay.git;a=atom']},
+	'pioto-overlay':{'feeds':['http://git.pioto.org/gitweb/pioto-overlay.git?a=atom']},
+	'thousand-parsec':{'feeds':['http://git.thousandparsec.net/gitweb/gitweb.cgi?p=gentoo-overlay.git;a=atom']},
+	'hollow':{'feeds':['http://git.xnull.de/gitweb/?p=overlay.git;a=atom']},
+	'zen-overlay':{'feeds':['http://git.zen-sources.org/?p=portage-overlay.git;a=atom']},
+	'THE':{'feeds':['http://git.zen-sources.org/?p=THE.git;a=atom']},
+	'lxde':{'feeds':['http://bitbucket.org/yngwin/lxde-overlay/atom/']},
+	'njw':{'feeds':['http://git.njw.me.uk/cgit/cgit.cgi/njw-gentoo-local/atom/?h=master']},
+	'oss-overlay':{'feeds':['http://hg.atheme.org/users/majeru/portage-overlay/atom-log']},
+	'devnull':{'feeds':['http://hg.core.ws/devnull/atom-log']},
+	'mamona':{'feeds':['http://rsalveti.net/git/?p=mamona-overlay;a=atom']},
+	'voyageur':{'feeds':['http://cafarelli.fr/websvn/rss.php?repname=voyageur-overlay&path=&isdir=1']},
+	'akoya':{'feeds':['https://hg.poildetroll.net/gentoo/overlay/atom-log']},
+	'pd-overlay':{'feeds':['http://sourceforge.net/export/rss2_keepsake.php?group_id=180376']},
+	'tryton':{'feeds':['http://hg.tryton.org/hgwebdir.cgi/tryton-overlay/atom-log']},
 }
 
 # List more or less from current <http://overlays.gentoo.org/>
diff --git a/feedextractors.py b/feedextractors.py
new file mode 100644
index 0000000..5bb0bea
--- /dev/null
+++ b/feedextractors.py
@@ -0,0 +1,57 @@
+import re
+FEED_EXTRACTORS = []
+
+def _register_feed_extractor(pattern, format):
+	regex = re.compile(pattern)
+	FEED_EXTRACTORS.append({'regex':regex, 'format':format})
+	
+
+# git://github.com/dang/kvm.git
+# http://github.com/feeds/dang/commits/kvm/master
+_register_feed_extractor(
+		'^[^ ]+://github.com/([^/]+)/([^/]+).git$',
+		'http://github.com/feeds/\\1/commits/\\2/master')
+
+
+# git://gitorious.org/gentoo-multimedia/gentoo-multimedia.git
+# http://gitorious.org/gentoo-multimedia.atom
+_register_feed_extractor(
+		'^[^ ]+://gitorious.org/([^/]+)/[^/]+.git$',
+		'http://gitorious.org/\\1.atom')
+
+# git://git.overlays.gentoo.org/dev/alexxy.git
+# http://git.overlays.gentoo.org/gitweb/?p=dev/alexxy.git;a=atom
+_register_feed_extractor(
+		'^[^ ]+://git.overlays.gentoo.org/(dev|proj)/([^/]+).git$',
+		'http://git.overlays.gentoo.org/gitweb/?p=\\1/\\2.git;a=atom')
+
+# git://repo.or.cz/dottout.git
+# http://repo.or.cz/w/dottout.git?a=rss
+_register_feed_extractor(
+		'^[^ ]+://repo.or.cz/([^/]+).git$',
+		'http://repo.or.cz/w/\\1.git?a=rss')  # atom broken currently
+
+# http://gentoo-china-overlay.googlecode.com/svn/trunk/
+# http://code.google.com/feeds/p/gentoo-china-overlay/svnchanges/basic
+_register_feed_extractor(
+		'^[^ ]+://([^ ]+).googlecode.com/svn/.*$',
+		'http://code.google.com/feeds/p/\\1/svnchanges/basic')
+
+# https://arcon.googlecode.com/hg/
+# http://code.google.com/feeds/p/arcon/hgchanges/basic
+_register_feed_extractor(
+		'^[^ ]+://([^ ]+).googlecode.com/hg/.*$',
+		'http://code.google.com/feeds/p/\\1/hgchanges/basic')
+		
+# git://git.goodpoint.de/overlay-sping.git
+# http://git.goodpoint.de/?p=overlay-sping.git;a=atom
+_register_feed_extractor(
+		'^[^ ]+://git.goodpoint.de/([^/]+).git$',
+		'http://git.goodpoint.de/?p=\\1.git;a=atom')
+
+
+# svn://overlays.gentoo.org/proj/toolchain
+# http://overlays.gentoo.org/proj/toolchain/timeline
+_register_feed_extractor(
+		'^svn://overlays.gentoo.org/proj/([^/]+)$',
+		'http://overlays.gentoo.org/proj/\\1/timeline')
diff --git a/layman-parser.py b/layman-parser.py
index dee43b2..e9c49f1 100755
--- a/layman-parser.py
+++ b/layman-parser.py
@@ -4,6 +4,7 @@ import xml.etree.ElementTree as ET
 import codecs
 
 from extradata import * # local
+from feedextractors import * # local
 
 def to_ascii(o, current_encoding='utf-8'):
     if not isinstance(o, basestring):
@@ -28,6 +29,11 @@ def recurse_print(e, level=0):
 	for c in e.getchildren():
 		recurse_print(c, level=level + 1)
 
+def append_feed(feed_uri, overlay_object):
+	feed = ET.Element('feed')
+	feed.text = feed_uri
+	overlay_object.append(feed)
+
 a = ET.parse(open('layman-global.txt'))
 overlays = a.getroot()
 
@@ -85,14 +91,17 @@ for overlay in overlays:
 		pass
 	
 	# Extend by feed URIs
+	for fe in FEED_EXTRACTORS:
+		uri = fe['regex'].sub(fe['format'], source.text)
+		if uri != source.text:
+			append_feed(uri, overlay)
+			continue
 	try:
 		feed_uris = extra_data['feeds']
 	except KeyError:
 		feed_uris = ()
 	for uri in feed_uris:
-		feed = ET.Element('feed')
-		feed.text = uri
-		overlay.append(feed)
+		append_feed(uri, overlay)
 	
 # Transform 'overlays' tag
 overlays.tag = 'repositories'