Initial move from Hyozan

.gitignore

@@ -0,0 +1,8 @@
+# JetBrains files
+.idea/*
+# Configuration
+conf.py
+# Data
+data/*
+auth_data.json
+files.db

Hyozan/db.py

@@ -0,0 +1,37 @@
+import sqlite3
+import time
+
+def connect(target):
+  return sqlite3.connect(target)
+
+def add_file(filename):
+  db = connect('files.db')
+  db.execute('INSERT INTO files (file, time, accessed) VALUES (?, ?, ?)',
+               [filename, time.time(), time.time()])
+  db.commit()
+  db.close()
+
+def update_file(filename):
+  db = connect('files.db')
+  db.execute('UPDATE files SET accessed = ? WHERE file = ?',
+               [time.time(), filename])
+  db.commit()
+  db.close()
+
+def add_b2(filename, file_id):
+  db = connect('files.db')
+  db.execute('UPDATE files SET b2 = ? WHERE file = ?',
+               [file_id, filename])
+  db.commit()
+  db.close()
+
+def check_value(column, value):
+  db = connect('files.db')
+  cur = db.execute('SELECT EXISTS(SELECT 1 FROM files WHERE ? = ?)', [column, value])
+  rv = cur.fetchone()
+  db.commit()
+  db.close()
+  if rv:
+    return False
+  else:
+    return True

Hyozan/output.py

@@ -0,0 +1,13 @@
+from datetime import datetime
+import time
+
+def print_log(source, message):
+  if source == "Main":
+    print('\033[92m' + source + ': \033[0m' + message)
+  elif source == "Notice" or source == "Warning":
+    print('\033[93m' + source + ': \033[0m' + message)
+  else:
+    print('\033[94m' + source + ': \033[0m' + message)
+
+def time_to_string(unixtime):
+  return datetime.fromtimestamp(unixtime).strftime('%B %d, %Y (%H:%M - ' + time.tzname[time.daylight] + ')')

README.md

@@ -0,0 +1,46 @@
+# Hyozan
+
+An "Object storage" like application which uses Backblaze B2 for archival. Originally intended for screenshot uploads via ShareX, but it will probably support other files as well.
+
+Named after the Japanese word 氷山 (Hyōzan), meaning iceberg. Referencing how most of it is hidden underwater.
+
+# The goal of this project
+
+Bandwidth is overpriced. Really overpriced.
+
+Don't get me wrong. B2's $0.05/GB is perfectly reasonable compared to all the others like S3 and Google Cloud. In fact, it's pretty good.
+
+Problem is that they're **all** overpriced. Storage is cheap, but never use these services for bandwidth alone.
+
+Why pay over $50 per TB of bandwidth when you can just install this on a VPS from a host like DigitalOcean that will give you the same for $5?
+
+# What it does
+
+When you upload a file to Hyozan, it forwards it to B2. When users then try to access it later, Hyozan will first check if it has a local copy. If it does not, it will fetch the file from B2 and keep it for a while.
+
+This will decrease both the bandwidth and transaction (request) costs that come with object storage services.
+
+## In technical terms
+
+Hyozan is an Object storage oriented API-only (for now) reverse proxy, designed to cache, manage, adding and hopefully deleting static files from a 3rd party object storage service while lowering your total bandwidth consumption from these services, in this case B2. Due to the code structure, rewriting it for something like S3 should not be too hard.
+
+# Why?
+
+Instead of paying $50 for 1 TB of B2 bandwidth. Let us assume that you have a DO droplet running Hyozan. If it cached 90% of your traffic. **That means you only pay about $5 for B2 bandwidth and $5 for your droplet. That's a total of just $10. A measly fifth of the regular cost**
+
+And this does not even consider transaction costs, which can be high if you serve a lot of smaller files.
+
+# Requirements
+
+* Python 3 (Python 2 might work, dunno, i don't test that, don't care either)
+* Install flask, currently that should be the only requirement and hopefully forever (``pip install -r requirements.txt``)
+
+# Using the thing
+
+* Clone the repo somewhere
+* Do ``cp conf.py.sample conf.py``
+* Edit ``conf.py`` so that the information is correct
+* If possible, make it listen on ``127.0.0.1`` and then use something like nginx as a reverse proxy. For security purposes
+* ``chmod +x run.py`` and then ``./run.py``
+* ???
+* PROFIT (Hopefully)

conf.py.sample

@@ -0,0 +1,25 @@
+# Create an empty config dict
+config = dict()
+##
+#
+#  Main server configuration
+#
+##
+config["HOST"] = "127.0.0.1"
+# This string will be used in file URLs that are returned
+config["DOMAIN"] = "example.com"
+config["PORT"] = 8282
+config["DEBUG"] = True
+# Extended debug will add extra debug output that's not normally provided by flask
+config["EXTENDED_DEBUG"] = False
+# Single user authentication, leave blank to disable authentication
+config["KEY"] = ""
+
+# File settings
+config["UPLOAD_FOLDER"] = './data'
+config["ALLOWED_EXTENSIONS"] = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])
+
+# Site info displayed to the user
+config["SITE_DATA"] = {
+  "title": "QuadFile"
+}

requirements.txt

@@ -0,0 +1,2 @@
+flask==0.10.1
+requests==2.9.1

run.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+from flask import Flask, Response, request, redirect, url_for, send_from_directory, abort, render_template
+from werkzeug import secure_filename
+from threading import Thread
+import logging
+import os
+import json
+import time
+from random import randint
+
+# Import our configuration
+from conf import config
+
+# Import Hyozan stuff
+from Hyozan import db
+from Hyozan.output import print_log, time_to_string
+
+app = Flask(__name__)
+
+
+# Pre-start functions
+print_log('Main', 'Running authorization towards B2')
+print_log('Main', 'Checking for data folder')
+if not os.path.exists(config['UPLOAD_FOLDER']):
+  print_log('Main', 'Data folder not found, creating')
+  os.makedirs(config['UPLOAD_FOLDER'])
+log = logging.getLogger('werkzeug')
+log.setLevel(logging.ERROR)
+
+
+def auth(key):
+  if config["KEY"] == "":
+    return True
+  elif config["KEY"] == key:
+    return True
+  else:
+    return False
+
+
+def allowed_file(filename):
+    return '.' in filename and filename.rsplit('.', 1)[1] in config["ALLOWED_EXTENSIONS"]
+
+
+@app.route('/', methods=['GET', 'POST'])
+def upload_file():
+  if request.method == 'POST':
+    if not auth(request.headers.get('X-Hyozan-Auth')):
+      abort(403)
+    data = dict()
+    file = request.files['file']
+
+    # Only continue if a file that's allowed gets submitted.
+    if file and allowed_file(file.filename):
+      filename = secure_filename(file.filename)
+      while os.path.exists(os.path.join(config["UPLOAD_FOLDER"], filename)):
+        filename = str(randint(1000,8999)) + '-' + secure_filename(filename)
+
+      thread1 = Thread(target = db.add_file, args = (filename,))
+      thread1.start()
+      print_log('Thread', 'Adding to DB')
+      file.save(os.path.join(config['UPLOAD_FOLDER'], filename))
+      #db.add_file(filename)
+      thread1.join()
+
+      data["file"] = filename
+      data["url"] = config["DOMAIN"] + "/" + filename
+      print_log('Main', 'New file processed')
+
+      if request.form["source"] == "web":
+        return redirect(url_for('get_file', filename=filename), code=302)
+      else:
+        return json.dumps(data)
+
+  # Return Web UI if we have a GET request
+  elif request.method == 'GET':
+    return render_template('upload.html', page=config["SITE_DATA"])
+
+
+@app.route('/<filename>', methods=['GET'])
+def get_file(filename):
+  print_log('Main', 'Hit "' + filename + '" - ' + time_to_string(time.time()))
+  return send_from_directory(config['UPLOAD_FOLDER'], filename)
+
+
+if __name__ == '__main__':
+  app.run(
+    port=config["PORT"],
+    host=config["HOST"],
+    debug=config["DEBUG"]
+  )

schema.sql

@@ -0,0 +1,8 @@
+-- noinspection SqlNoDataSourceInspectionForFile
+drop table if exists files;
+create table files (
+  file text primary key not null,
+  b2 text,
+  time int,
+  accessed int
+);

static/style.css

@@ -0,0 +1,69 @@
+@import url(https://fonts.googleapis.com/css?family=Lato:700,300,400,100);
+
+/* Being lazy 101 */
+* {
+  margin: 0;
+  padding: 0;
+  box-sizing: border-box;
+}
+
+body {
+  font-family: "Lato", sans-serif;
+  color: #ffffff;
+  background-color: #101010;
+}
+
+.header {
+  width: 100%;
+  padding: 10px;
+}
+
+.header h1 {
+  font-size: 36pt;
+  font-weight: 300;
+  text-align: center;
+}
+
+.header .inner {
+  width: 100%;
+  max-width: 920px;
+  margin-left: auto;
+  margin-right: auto;
+}
+
+.page {
+  width: 100%;
+  max-width: 920px;
+  padding: 10px;
+  margin-left: auto;
+  margin-right: auto;
+}
+
+.uploadForm {
+  width: 100%;
+  text-align: center;
+}
+
+.uploadForm input[type="file"] {
+  padding: 10px;
+  font-size: 12pt;
+  font-weight: 300;
+  border: none;
+  display: inline-block;
+}
+
+.uploadForm input[type="submit"] {
+  width: 100%;
+  padding: 20px;
+  font-size: 12pt;
+  font-weight: 300;
+  max-width: 320px;
+  background-color: #229922;
+  margin-top: 10px;
+  color: #FFFFFF;
+  border: none;
+}
+
+.uploadForm input[type="submit"]:hover {
+  background-color: #33BB33;
+}

templates/layout.html

@@ -0,0 +1,21 @@
+<!doctype html>
+<html>
+  <head>
+    <title>{{ page.title }}</title>
+    <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css') }}">
+    <link rel="shortcut icon" href="{{ url_for('static', filename='favicon.ico') }}">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+  </head>
+  <body>
+    <div class="header">
+      <div class="inner">
+        <h1>
+          {{ page.title }}
+        </h1>
+      </div>
+    </div>
+    <div class="page">
+      {% block body %}{% endblock %}
+    </div>
+  </body>
+</html>

templates/upload.html

@@ -0,0 +1,12 @@
+{% extends "layout.html" %}
+{% block body %}
+  <form action="" id="form" method="post" enctype="multipart/form-data" class="uploadForm">
+    <input id="file" type="file" name="file">
+    <input type="hidden" name="source" value="web">
+  </form>
+  <script>
+  document.getElementById("file").onchange = function() {
+    document.getElementById("form").submit();
+  };
+  </script>
+{% endblock %}