Initial commit

10 files changed, 370 insertions, 0 deletions

diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..6a287d6
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,13 @@
+Copyright (c) 2018, George Tokmaji
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..124e6e8
--- /dev/null
+++ b/README.md
@@ -0,0 +1,2 @@
+# Parry
+A larry-like server written in Python
diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/__init__.py
diff --git a/__main__.py b/__main__.py
new file mode 100644
index 0000000..f46e94c
--- /dev/null
+++ b/__main__.py
@@ -0,0 +1,32 @@
+# Copyright (c) 2018, George Tokmaji
+
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from .helpers import *
+from .handlers import *
+from .routes import *
+
+@route("/api")
+def get_root():
+    notAllowed()
+
+if __name__ == "__main__":
+    def loadEntries():
+        for handler in [CC(), CCAN()]:
+            for entry in handler.process():
+                database["entries"][entry["id"]] = entry
+                saveJSON(database, "database.json")
+    
+    threading.Thread(target=loadEntries, daemon=True).start()
+    run(host='0.0.0.0', port=9621, debug=True)
+    saveJSON(database, "database.json")
diff --git a/handlers.py b/handlers.py
new file mode 100644
index 0000000..8139c44
--- /dev/null
+++ b/handlers.py
@@ -0,0 +1,132 @@
+# Copyright (c) 2018, George Tokmaji
+
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from .helpers import *
+import locale, html, string
+from datetime import datetime
+from abc import *
+
+class Site(ABC):
+    regex = None
+    
+    @abstractmethod
+    def process(self):
+        return
+
+class Parry(Site):
+    def process(self):
+        yield {}
+
+class Website(Site):
+    tag = ""
+    prefix = ""
+    address = ""
+    date_format = ""
+    regexes = {
+        "list" : "",
+        "desc" : ""
+            }
+    
+    def process(self):
+        r = requests.get(self.address)
+        if r:
+            i = 0
+            for m in self.regexes["list"].finditer(r.text):
+                #if i > 10:
+                    #break
+                print(m["title"])
+                id = str(ObjectId.from_datetime(datetime.strptime(m["updatedAt"], self.date_format)))
+                if id not in database["entries"]:
+                    entry = {
+                        "title" : html.unescape(m["title"]),
+                        "voting" : {
+                            "sum" : round(float(m["niveau"]), 0) if "niveau" in m.groups() else 0,
+                            "count" : 0,
+                            "votes" : None
+                            },
+                        "tags" : [self.tag],
+                        "files" : [],
+                        "dependencies" : [],
+                        "deleted" : False,
+                        "description" : "",
+                        "pic" : None,
+                        "author" : {
+                            "username" : html.unescape(m["author"])
+                            },
+                        "slug" : "",
+                        "updatedAt" : datetime.strptime(m["updatedAt"], self.date_format).isoformat(),
+                        "__v" : 1,
+                        "comments" : None,
+                        "id" : id,
+                        "__intern" : {
+                            "entryURL" : self.prefix + m["entryURL"]
+                            }
+                        }
+                    
+                    downloadURL = self.prefix + m["downloadURL"]
+                    r = requests.get(downloadURL, stream=True, allow_redirects=True)
+                    if not r:
+                        continue
+                    
+                    locale.setlocale(locale.LC_ALL, "C")
+                    entry["files"] = [{
+                            "metadata" : {
+                                "hashes" : {
+                                    "sha1" : calculateHashForResource(r).hexdigest()
+                                    },
+                                "downloadURL" : downloadURL
+                                },
+                            "aliases" : None,
+                            "deleted" : False,
+                            "_id" : entry["id"],
+                            "filename" : m["downloadURL"].split("/")[-1],
+                            "content-type" : r.headers.get("Content-Type", "application/octet-stream"),
+                            "length" : int(r.headers["Content-Length"]),
+                            "chunkSize" : 4096, # what is this for
+                            "uploadDate" : datetime.strptime(r.headers["Date"], "%a, %d %b %Y %H:%M:%S GMT").isoformat(),
+                            }
+                        ]
+                    
+                    locale.setlocale(locale.LC_ALL, "")
+                
+                    entry["createdAt"] = entry["updatedAt"]
+                    entry["slug"] = "".join(filter(lambda x: x in string.ascii_letters, entry["title"].lower()))
+                    
+                    r = requests.get(entry["__intern"]["entryURL"])
+                    d = self.regexes["desc"].match(r.text)
+                    if d and "description" in d.groups():
+                        entry["description"] = html.unescape(d["description"])
+                    yield entry
+                i += 1
+
+class CCAN(Website):
+    regexes = {
+        "list" : re.compile(r"<TR.*?><.*?><IMG SRC=\"/img/type-(?P<type>.*?)\.gif\".*?<A HREF=\"(?P<entryURL>ccan-view\.pl\?a=view\&i=\d*?)\">(?P<title>.*?)(<I>v</I>(?P<version>.*?))?</A><.*?><A HREF=\"(?P<downloadURL>ccan-dl-auth\.pl/(?P<id>\d*)/.*?)\"><.*?><A HREF=\"ccan-user.pl.*?\">(?P<author>.*?)</A><.*?>\((?P<niveau>\d\.\d)\).*?>(?P<updatedAt>\d\d\.\d\d\.\d\d\ \d\d\:\d\d).*?</TR>"),
+        "desc" : re.compile(r"<TD>Beschreibung:</TD><TD>(?P<description>.*?)</TD></TR>")
+        }
+    
+    address = "https://ccan.de/cgi-bin/ccan/ccan-view.pl?a=&sc=tm&so=d&nr=100000&pg=0&ac=ty-ti-ni-tm-rp-ev&reveal=1"
+    tag = "ccan"
+    prefix = "https://ccan.de/cgi-bin/ccan/"
+    date_format = "%d.%m.%y %H:%M"
+
+class CC(Website):
+    regexes = {
+        "list" : re.compile(r"<tr><td align=\"right\">.*?<a href=\"/(?P<entryURL>download\.php\?act=getinfo&amp;dl=\d*?)\">(?P<title>.*?)</a></td><td align=\"right\"><a href=\"(?P<downloadURL>downloads/dl\d*?/.*?)\"><img src=\"picz/dl\.gif\" alt=\"Runterladen\" title=\"Runterladen\" border=\"0\"></a></td><td>(<a href=\"user\.php.*?\">|)(?P<author>.*?)(</a>|)</td>.*?<td style=\"border-right:0px;\">(?P<updatedAt>\d\d\.\d\d\.\d\d\d\d \d{1,2}:\d\d)</td></tr>"),
+        "desc" : re.compile("")
+        }
+    address = "https://cc-archive.lwrl.de/download.php"
+    tag = "clonk-center"
+    prefix = "https://cc-archive.lwrl.de/"
+    date_format = "%d.%m.%Y %H:%M"
diff --git a/helpers.py b/helpers.py
new file mode 100644
index 0000000..0e9da70
--- /dev/null
+++ b/helpers.py
@@ -0,0 +1,57 @@
+# Copyright (c) 2018, George Tokmaji
+
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import sys
+import os, re, json, math
+import requests, hashlib
+from bottle import route, run, Bottle, request, static_file, response, hook, HTTPResponse
+from bson.objectid import ObjectId
+import threading
+
+os.chdir(os.path.dirname(__file__))
+
+io_lock = threading.Lock()
+
+def loadJSON(name : str) -> dict:
+    try:
+        with open(name, "r") as fobj:
+            return json.load(fobj)
+    
+    except (FileNotFoundError, json.decoder.JSONDecodeError):
+        return {}
+
+def saveJSON(obj : dict, name : str) -> None:
+    with io_lock:
+        with open(name, "w") as fobj:
+            json.dump(obj, fobj)
+
+database = loadJSON("database.json")
+if "entries" not in database:
+    database["entries"] = {}
+
+def calculateHashForResource(resource : requests.Response, hashobj : object = None) -> object:
+    if hashobj is None:
+        hashobj = hashlib.sha1()
+    
+    l = 0
+    for block in resource.iter_content(4096):
+        l += len(block)
+        hashobj.update(block)
+    
+    if "content-length" not in resource.headers:
+        resource.headers["Content-Length"] = l
+    return hashobj
+
+def notAllowed():
+    raise HTTPResponse(f"Cannot {request.method} {request.path}")
diff --git a/routes/__init__.py b/routes/__init__.py
new file mode 100644
index 0000000..a92e40a
--- /dev/null
+++ b/routes/__init__.py
@@ -0,0 +1,17 @@
+# Copyright (c) 2018, George Tokmaji
+
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from .uploads import *
+from .media import *
+from .auth import *
diff --git a/routes/auth.py b/routes/auth.py
new file mode 100644
index 0000000..ebf52c0
--- /dev/null
+++ b/routes/auth.py
@@ -0,0 +1,19 @@
+# Copyright (c) 2018, George Tokmaji
+
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from ..helpers import *
+
+@route("/api/auth", method="POST")
+def post_auth():
+    raise HTTPResponse(status=501)
diff --git a/routes/media.py b/routes/media.py
new file mode 100644
index 0000000..6c3c831
--- /dev/null
+++ b/routes/media.py
@@ -0,0 +1,49 @@
+# Copyright (c) 2018, George Tokmaji
+
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from ..helpers import *
+from datetime import datetime
+
+@route("/api/media")
+def get_media():
+    notAllowed()
+
+@route("/api/media", method="POST")
+def post_media():
+    try:
+        entry = database["entries"][request.forms["id"]]
+    except KeyError as e:
+        raise HTTPResponse("Invalid id", 400) from e
+    
+    for f in request.files:
+        entry["files"].append({})
+            
+    id = ObjectId()
+
+@route("/api/media/<id>")
+def get_media_id(id):
+    for entry in database["entries"].values():
+        for file in entry["files"]:
+            if file["_id"] == id:
+                response.set_header("Content-Type", file["content-type"])
+                response.set_header("Content-Length", file["length"])
+                response.set_header("Date", datetime.fromisoformat(file["uploadDate"]).strftime("%a, %d %b %Y %H:%M:%S GMT"))
+                response.set_header("Content-Disposition", f"attachment; filename=\"{file['filename']}\"")
+                if request.method == "GET":
+                    if "downloadURL" in file["metadata"]:
+                        return HTTPResponse(status=302, headers={"Location" : file["metadata"]["downloadURL"]})
+                    else:
+                        return static_file(file["filename"], root=f"media/{id}")
+    
+    raise HTTPResponse(status=404)
diff --git a/routes/uploads.py b/routes/uploads.py
new file mode 100644
index 0000000..703d3d1
--- /dev/null
+++ b/routes/uploads.py
@@ -0,0 +1,49 @@
+# Copyright (c) 2018, George Tokmaji
+
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from ..helpers import *
+
+@route("/api/uploads")
+def get_uploads():
+    ret = {
+        "pagination" : {
+            "total" : 0,
+            "limit" : 50,
+            "page" : 1,
+            "pages" : 1
+            },
+        "uploads" : []
+        }
+    
+    for entry in database["entries"].values():
+        if "__intern" in entry:
+            entry = entry.copy()
+            del entry["__intern"]
+
+        ret["uploads"].append(entry)
+    
+    ret["pagination"]["total"] = ret["pagination"]["limit"] = len(ret["uploads"])
+    
+    return ret
+
+@route("/api/uploads/<id>")
+def get_upload(id):
+    if id in database["entries"]:
+        return database["entries"][id]
+    else:
+        raise HTTPResponse(status=404)
+
+@route("/api/uploads", method="POST")
+def post_upload():
+    raise HTTPResponse(status=501)