diff options
Diffstat (limited to 'handlers.py')
| -rw-r--r-- | handlers.py | 132 |
1 files changed, 132 insertions, 0 deletions
diff --git a/handlers.py b/handlers.py new file mode 100644 index 0000000..8139c44 --- /dev/null +++ b/handlers.py @@ -0,0 +1,132 @@ +# Copyright (c) 2018, George Tokmaji + +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. + +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from .helpers import * +import locale, html, string +from datetime import datetime +from abc import * + +class Site(ABC): + regex = None + + @abstractmethod + def process(self): + return + +class Parry(Site): + def process(self): + yield {} + +class Website(Site): + tag = "" + prefix = "" + address = "" + date_format = "" + regexes = { + "list" : "", + "desc" : "" + } + + def process(self): + r = requests.get(self.address) + if r: + i = 0 + for m in self.regexes["list"].finditer(r.text): + #if i > 10: + #break + print(m["title"]) + id = str(ObjectId.from_datetime(datetime.strptime(m["updatedAt"], self.date_format))) + if id not in database["entries"]: + entry = { + "title" : html.unescape(m["title"]), + "voting" : { + "sum" : round(float(m["niveau"]), 0) if "niveau" in m.groups() else 0, + "count" : 0, + "votes" : None + }, + "tags" : [self.tag], + "files" : [], + "dependencies" : [], + "deleted" : False, + "description" : "", + "pic" : None, + "author" : { + "username" : html.unescape(m["author"]) + }, + "slug" : "", + "updatedAt" : datetime.strptime(m["updatedAt"], self.date_format).isoformat(), + "__v" : 1, + "comments" : None, + "id" : id, + "__intern" : { + "entryURL" : self.prefix + m["entryURL"] + } + } + + downloadURL = self.prefix + m["downloadURL"] + r = requests.get(downloadURL, stream=True, allow_redirects=True) + if not r: + continue + + locale.setlocale(locale.LC_ALL, "C") + entry["files"] = [{ + "metadata" : { + "hashes" : { + "sha1" : calculateHashForResource(r).hexdigest() + }, + "downloadURL" : downloadURL + }, + "aliases" : None, + "deleted" : False, + "_id" : entry["id"], + "filename" : m["downloadURL"].split("/")[-1], + "content-type" : r.headers.get("Content-Type", "application/octet-stream"), + "length" : int(r.headers["Content-Length"]), + "chunkSize" : 4096, # what is this for + "uploadDate" : datetime.strptime(r.headers["Date"], "%a, %d %b %Y %H:%M:%S GMT").isoformat(), + } + ] + + locale.setlocale(locale.LC_ALL, "") + + entry["createdAt"] = entry["updatedAt"] + entry["slug"] = "".join(filter(lambda x: x in string.ascii_letters, entry["title"].lower())) + + r = requests.get(entry["__intern"]["entryURL"]) + d = self.regexes["desc"].match(r.text) + if d and "description" in d.groups(): + entry["description"] = html.unescape(d["description"]) + yield entry + i += 1 + +class CCAN(Website): + regexes = { + "list" : re.compile(r"<TR.*?><.*?><IMG SRC=\"/img/type-(?P<type>.*?)\.gif\".*?<A HREF=\"(?P<entryURL>ccan-view\.pl\?a=view\&i=\d*?)\">(?P<title>.*?)(<I>v</I>(?P<version>.*?))?</A><.*?><A HREF=\"(?P<downloadURL>ccan-dl-auth\.pl/(?P<id>\d*)/.*?)\"><.*?><A HREF=\"ccan-user.pl.*?\">(?P<author>.*?)</A><.*?>\((?P<niveau>\d\.\d)\).*?>(?P<updatedAt>\d\d\.\d\d\.\d\d\ \d\d\:\d\d).*?</TR>"), + "desc" : re.compile(r"<TD>Beschreibung:</TD><TD>(?P<description>.*?)</TD></TR>") + } + + address = "https://ccan.de/cgi-bin/ccan/ccan-view.pl?a=&sc=tm&so=d&nr=100000&pg=0&ac=ty-ti-ni-tm-rp-ev&reveal=1" + tag = "ccan" + prefix = "https://ccan.de/cgi-bin/ccan/" + date_format = "%d.%m.%y %H:%M" + +class CC(Website): + regexes = { + "list" : re.compile(r"<tr><td align=\"right\">.*?<a href=\"/(?P<entryURL>download\.php\?act=getinfo&dl=\d*?)\">(?P<title>.*?)</a></td><td align=\"right\"><a href=\"(?P<downloadURL>downloads/dl\d*?/.*?)\"><img src=\"picz/dl\.gif\" alt=\"Runterladen\" title=\"Runterladen\" border=\"0\"></a></td><td>(<a href=\"user\.php.*?\">|)(?P<author>.*?)(</a>|)</td>.*?<td style=\"border-right:0px;\">(?P<updatedAt>\d\d\.\d\d\.\d\d\d\d \d{1,2}:\d\d)</td></tr>"), + "desc" : re.compile("") + } + address = "https://cc-archive.lwrl.de/download.php" + tag = "clonk-center" + prefix = "https://cc-archive.lwrl.de/" + date_format = "%d.%m.%Y %H:%M" |