venus/aperisolve
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Huge refactoring

Browse Source
This commit is contained in:
Zeecka
2026-01-08 05:34:24 +01:00
parent 4b6a944817
commit 5719ee7c3e
15 changed files with 424 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+1 -2
View File
@@ -1,5 +1,5 @@
# Application configuration
MAX_CONTENT_LENGTH=1048576
MAX_CONTENT_LENGTH=3145728
MAX_PENDING_TIME=600
MAX_STORE_TIME=259200
CLEAR_AT_RESTART=1
@@ -7,7 +7,6 @@ SKIP_IHDR_FILL=0
# Flask configuration
FLASK_DEBUG=0
FLASK_ENV=development
# Redis configuration
RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0
Dockerfile
+1 -1
View File
@@ -49,4 +49,4 @@ COPY aperisolve/ /app/aperisolve/
RUN pip install --no-cache-dir -r /app/aperisolve/requirements.txt
# Commande de lancement
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "--access-logfile", "-", "--error-logfile", "-", "--log-level", "info", "--capture-output", "aperisolve.wsgi:application"]
CMD ["gunicorn", "-w", "8", "-b", "0.0.0.0:5000", "--access-logfile", "-", "--error-logfile", "-", "--log-level", "info", "--capture-output", "aperisolve.utils.wsgi:application"]
aperisolve/__init__.py
+3
View File
@@ -1,3 +1,6 @@
# flake8: noqa: E203,E501,W503
# pylint: disable=C0413,W0718,R0903,R0801
# mypy: disable-error-code=unused-awaitable
from . import app, config, models, workers
__all__ = ["app", "config", "models", "workers"]
aperisolve/analyzers/base_analyzer.py
+2 -1
View File
@@ -13,8 +13,9 @@ from pathlib import Path
from shutil import rmtree
from typing import Any, Optional, overload
from ..config import MAX_PENDING_TIME
_thread_lock = threading.Lock()
MAX_PENDING_TIME = int(os.getenv("MAX_PENDING_TIME", "600")) # 10 minutes by default
class SubprocessAnalyzer(ABC):
aperisolve/app.py
+14 -29
View File
@@ -16,28 +16,23 @@ from flask import Flask, Response, abort, jsonify, render_template, request, sen
from redis import Redis
from rq import Queue
from .sentry import initialize_sentry
from .utils.sentry import initialize_sentry
initialize_sentry()
from .cleanup import cleanup_old_entries
from .config import IMAGE_EXTENSIONS, RESULT_FOLDER, WORKER_FILES
from .models import Image, Submission, db, init_db
from .models import Image, Submission, cleanup_old_entries, db
app: Flask = Flask(__name__)
def create_app() -> Flask:
"""Create flask application with routes."""
app = Flask(__name__)
app.json.sort_keys = False # type: ignore
app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("DB_URI")
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
app.config["MAX_CONTENT_LENGTH"] = int(os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024))
RESULT_FOLDER.mkdir(parents=True, exist_ok=True)
app.config["REDIS_QUEUE"] = Queue(connection=Redis(host="redis", port=6379))
db.init_app(app)
with app.app_context():
init_db(app)
redis_conn = Redis(host="redis", port=6379)
queue = Queue(connection=redis_conn)
@app.errorhandler(413)
def too_large(_: Any) -> tuple[Response, int]:
@@ -52,28 +47,21 @@ def too_large(_: Any) -> tuple[Response, int]:
sentry_sdk.capture_message("Upload failed: 413 Payload Too Large", level="warning")
return jsonify({"error": "Image size exceeded"}), 413
@app.errorhandler(404)
def not_found(_: Any) -> str:
"""Error Handler for 404 not found."""
return render_template("error.html", message="Resource not found", statuscode=404)
# Sentry automatically captures 500 errors
@app.route("/")
def index() -> str:
"""Render the main index page."""
return render_template("index.html")
@app.route("/faq")
def faq() -> str:
"""Render the FAQ page."""
return render_template("faq.html")
@app.route("/show")
def show() -> str:
"""Show all active submissions."""
@@ -84,7 +72,6 @@ def show() -> str:
images.append({"file": f"/image/{img.hash}", "link": f"/{last_sub.hash}"})
return render_template("show.html", images=images)
@app.route("/<hash_val>", methods=["GET"])
def result_page(hash_val: str) -> str:
"""Render the result page for a given submission hash."""
@@ -93,7 +80,6 @@ def result_page(hash_val: str) -> str:
return render_template("result.html", hash_val=hash_val)
@app.route("/upload", methods=["POST"])
def upload_image() -> tuple[Response, int]:
"""Handle image upload and initiate analysis."""
@@ -186,19 +172,19 @@ def upload_image() -> tuple[Response, int]:
db.session.commit() # pylint: disable=no-member
# Start the analysis jobs
queue.enqueue("aperisolve.workers.analyze_image", submission.hash, job_timeout=300)
app.config["REDIS_QUEUE"].enqueue(
"aperisolve.workers.analyze_image", submission.hash, job_timeout=300
)
# Return submission identifier
return jsonify({"submission_hash": submission.hash}), 200
@app.route("/status/<hash_val>", methods=["GET"])
def get_status(hash_val: str) -> Response:
"""Get the processing status of a submission."""
submission = Submission.query.get_or_404(hash_val) # type: ignore
return jsonify({"status": submission.status}) # type: ignore
@app.route("/infos/<hash_val>", methods=["GET"])
def get_infos(hash_val: str) -> Response:
"""Get the metadata and information of a submission."""
@@ -218,7 +204,6 @@ def get_infos(hash_val: str) -> Response:
}
) # type: ignore
@app.route("/result/<hash_val>", methods=["GET"])
def get_result(hash_val: str) -> tuple[Response, int]:
"""Get the analysis results of a submission."""
@@ -235,7 +220,6 @@ def get_result(hash_val: str) -> tuple[Response, int]:
return jsonify({"results": results}), 200
@app.route("/download/<hash_val>/<tool>")
def download_output(hash_val: str, tool: str) -> Response:
"""Download the output of a specific analyzer for a given submission hash."""
@@ -251,7 +235,6 @@ def download_output(hash_val: str, tool: str) -> Response:
return send_file(output_file, as_attachment=True)
@app.route("/image/<img_name>")
@app.route("/image/<hash_val>/<img_name>")
def get_image(hash_val: Optional[str] = None, img_name: Optional[str] = None) -> Response:
@@ -279,7 +262,9 @@ def get_image(hash_val: Optional[str] = None, img_name: Optional[str] = None) ->
return send_file(output_file, as_attachment=True)
return app
if __name__ == "__main__":
debug_mode = os.environ.get("FLASK_DEBUG", "false").lower() == "true"
app.run(host="0.0.0.0", port=5000, debug=debug_mode)
my_app = create_app()
debug_mode = os.environ.get("FLASK_DEBUG", "0") == "1"
my_app.run(host="0.0.0.0", port=5000, debug=debug_mode)
aperisolve/cleanup.py
-79
View File
@@ -1,79 +0,0 @@
# flake8: noqa: E203,E501,W503
# pylint: disable=C0413,W0718,R0903,R0801
# mypy: disable-error-code=unused-awaitable
"""Cleanup old entries in the database and file system."""
import shutil
import time
from datetime import datetime, timezone
from os import getenv
from pathlib import Path
from .models import Image, Submission, db
RESULT_FOLDER = Path("./results")
MAX_PENDING_TIME = int(getenv("MAX_PENDING_TIME", "600")) # 10 minutes by default
MAX_STORE_TIME = int(getenv("MAX_STORE_TIME", "259200")) # 3 days by default
def cleanup_old_entries() -> None:
"""
Clean up old and incomplete entries from the database and file system.
This function performs two main cleanup operations:
1. Submission cleanup:
- Deletes submissions with "pending" or "running" status that have exceeded
the maximum allowed processing time (MAX_PENDING_TIME).
- Deletes completed submissions ("done" status) that have missing or buggy
results (results.json file not found).
2. Image cleanup:
- Deletes images older than MAX_STORE_TIME along with all associated
submissions and their result folders.
- Deletes orphaned images (with no submissions) older than MAX_PENDING_TIME
and removes their associated result folders from the file system.
All database changes are committed at the end of the operation.
Returns:
None
"""
now = time.time()
for submission in Submission.query.all(): # type: ignore
if (
submission.status in ("pending", "running") # type: ignore
and now - submission.date > MAX_PENDING_TIME # type: ignore
):
# Processing took too long, delete
db.session.delete(submission) # pylint: disable=no-member
elif submission.status == "done": # type: ignore
# Search for buggy results, delete
result_path = RESULT_FOLDER / str(submission.image_hash) / str(submission.hash)
result_file = result_path / "results.json"
if not result_file.exists():
db.session.delete(submission) # pylint: disable=no-member
shutil.rmtree(result_path)
# Delete "old" images
for img in Image.query.all(): # type: ignore
if img.last_submission_date.tzinfo is None:
img_date = img.last_submission_date.replace(tzinfo=timezone.utc)
else:
img_date = img.last_submission_date
delay = datetime.now(timezone.utc) - img_date
img_fold = RESULT_FOLDER / img.hash
if delay.total_seconds() > MAX_STORE_TIME:
for s in img.submissions:
db.session.delete(s) # pylint: disable=no-member
if img_fold.exists():
shutil.rmtree(img_fold) # type: ignore
db.session.delete(img) # pylint: disable=no-member
# Delete Images with missing submission
if len(img.submissions) == 0 and delay.total_seconds() > MAX_PENDING_TIME:
if img_fold.exists():
shutil.rmtree(img_fold) # type: ignore
db.session.delete(img) # pylint: disable=no-member
db.session.commit() # pylint: disable=no-member
aperisolve/config.py
+4
View File
@@ -3,9 +3,13 @@
# mypy: disable-error-code=unused-awaitable
"""Aperi'Solve configuration variables."""
from os import getenv
from pathlib import Path
IMAGE_EXTENSIONS = [".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".tiff"]
WORKER_FILES = ["binwalk", "foremost", "steghide", "zsteg", "image_resize", "openstego", "pcrt"]
RESULT_FOLDER = Path(__file__).parent.resolve() / "results"
MAX_PENDING_TIME = int(getenv("MAX_PENDING_TIME", "600")) # 10 minutes by default
MAX_STORE_TIME = int(getenv("MAX_STORE_TIME", "259200")) # 3 days by default
CLEAR_AT_RESTART = int(getenv("CLEAR_AT_RESTART", "0"))
aperisolve/models.py
+74 -77
View File
@@ -1,18 +1,15 @@
# flake8: noqa: E203,E501,W503
# pylint: disable=C0413,W0718,R0903,R0801
# pylint: disable=E1101,C0413,W0212,W0718,R0903,R0801
# mypy: disable-error-code=unused-awaitable
"""This module defines the database models for the Aperi'Solve application."""
import itertools
import shutil
import struct
import sys
import time
import zlib
from datetime import datetime, timezone
from os import getenv
from pathlib import Path
from shutil import rmtree
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy import (
BigInteger,
@@ -25,6 +22,7 @@ from sqlalchemy import (
String,
)
from aperisolve.config import MAX_PENDING_TIME, MAX_STORE_TIME, RESULT_FOLDER
from aperisolve.utils.utils import get_resolutions, get_valid_depth_color_pairs
db: SQLAlchemy = SQLAlchemy()
@@ -117,9 +115,10 @@ def fill_ihdr_db() -> None:
Computes CRC for each combination and stores parameters directly.
"""
# Skip if table already has entries
if db.session.query(IHDR.iid).first() is not None:
try:
# Skip if table already has entries (check before doing any work)
count_query = db.session.query(IHDR.iid).limit(1)
if db.session.execute(count_query).first() is not None:
print("IHDR table already populated, skipping fill.")
return
@@ -132,95 +131,93 @@ def fill_ihdr_db() -> None:
bit_color_pairs,
interlace_methods,
)
batch = []
batch_size = 10000 # Increased from 5000
count = 0
with db.session.no_autoflush:
for (width, height), (bit_depth, color_type), interlace in combinations:
# Compute CRC directly
crc = IHDR.compute_crc(width, height, bit_depth, color_type, interlace)
# Check if entry already exists
exists = db.session.execute(
db.select(IHDR.iid).where(
db.and_(
IHDR.crc == crc,
IHDR.width == width,
IHDR.height == height,
IHDR.bit_depth == bit_depth,
IHDR.color_type == color_type,
IHDR.interlace == interlace,
# Add new entry
db.session.add(
IHDR(
crc=crc,
width=width,
height=height,
bit_depth=bit_depth,
color_type=color_type,
interlace=interlace,
)
)
).first()
if exists:
continue
batch.append(
{
"crc": crc,
"width": width,
"height": height,
"bit_depth": bit_depth,
"color_type": color_type,
"interlace": interlace,
}
)
count += 1
if len(batch) >= batch_size:
db.session.bulk_insert_mappings(IHDR, batch)
db.session.commit()
if count % 5000 == 0:
print(f"Inserted {count} IHDR entries...")
batch = []
# Insert remaining entries
if batch:
db.session.bulk_insert_mappings(IHDR, batch)
db.session.commit()
print(f"Inserted {count} total IHDR entries.")
except Exception as e:
print(f"Error filling IHDR table: {e}")
db.session.rollback()
def init_db(app: Flask) -> None:
def cleanup_old_entries() -> None:
"""
Initialize the database by creating tables and populating lookup data.
Clean up old and incomplete entries from the database and file system.
This function creates the database schema if it doesn't already exist,
and populates the CRC lookup table for PNG IHDR validation.
This function performs two main cleanup operations:
Args:
app (Flask): The Flask application instance used to establish the application context
for database operations.
1. Submission cleanup:
- Deletes submissions with "pending" or "running" status that have exceeded
the maximum allowed processing time (MAX_PENDING_TIME).
- Deletes completed submissions ("done" status) that have missing or buggy
results (results.json file not found).
2. Image cleanup:
- Deletes images older than MAX_STORE_TIME along with all associated
submissions and their result folders.
- Deletes orphaned images (with no submissions) older than MAX_PENDING_TIME
and removes their associated result folders from the file system.
All database changes are committed at the end of the operation.
Returns:
None
Side Effects:
- Creates all database tables if the "image" table doesn't exist
- Populates the IHDR CRC lookup table with initial data
- Prints status messages to console during initialization
"""
# Detect if running as RQ worker by checking command line
# Worker: /usr/local/bin/rq worker ...
# Web: /usr/local/bin/flask run ... or gunicorn/wsgi
is_worker = any("rq" in arg and "flask" not in arg for arg in sys.argv[:2])
now = time.time()
for submission in Submission.query.all(): # type: ignore
if (
submission.status in ("pending", "running") # type: ignore
and now - submission.date > MAX_PENDING_TIME # type: ignore
):
# Processing took too long, delete
db.session.delete(submission) # pylint: disable=no-member
elif submission.status == "done": # type: ignore
# Search for buggy results, delete
result_path = RESULT_FOLDER / str(submission.image_hash) / str(submission.hash)
result_file = result_path / "results.json"
if not result_file.exists():
db.session.delete(submission) # pylint: disable=no-member
shutil.rmtree(result_path)
with app.app_context():
# Workers should never clear the database
if not is_worker and getenv("CLEAR_AT_RESTART", "0") == "1":
print("Clearing database and file system at restart...")
db.session.remove() # pylint: disable=no-member
db.drop_all()
rmtree(Path("./results"), ignore_errors=True) # Clear results folder
# Always create tables (idempotent - safe to call multiple times)
db.create_all()
print("Database structure created successfully.")
# Workers should never fill IHDR table
if is_worker:
print("Running as worker, skipping IHDR table fill.")
# Delete "old" images
for img in Image.query.all(): # type: ignore
if img.last_submission_date.tzinfo is None:
img_date = img.last_submission_date.replace(tzinfo=timezone.utc)
else:
fill_ihdr_db()
img_date = img.last_submission_date
delay = datetime.now(timezone.utc) - img_date
img_fold = RESULT_FOLDER / img.hash
if delay.total_seconds() > MAX_STORE_TIME:
for s in img.submissions:
db.session.delete(s) # pylint: disable=no-member
if img_fold.exists():
shutil.rmtree(img_fold) # type: ignore
db.session.delete(img) # pylint: disable=no-member
# Delete Images with missing submission
if len(img.submissions) == 0 and delay.total_seconds() > MAX_PENDING_TIME:
if img_fold.exists():
shutil.rmtree(img_fold) # type: ignore
db.session.delete(img) # pylint: disable=no-member
db.session.commit() # pylint: disable=no-member
aperisolve/utils/init_db.py
+62
View File
@@ -0,0 +1,62 @@
"""
Database initialization script.
This script is meant to be run ONCE at deploy time, never from Gunicorn or runtime code.
If CLEAR_AT_RESTART is set, database will be reset at launch time.
"""
import os
from shutil import rmtree
from sqlalchemy import inspect
from ..app import create_app
from ..config import RESULT_FOLDER
from ..models import db, fill_ihdr_db
def main() -> None:
"""Database initialization main function."""
reset = os.getenv("CLEAR_AT_RESTART") == "1"
app = create_app()
with app.app_context():
inspector = inspect(db.engine)
tables = inspector.get_table_names()
if reset:
# Delete result folder content
if RESULT_FOLDER.exists():
for item in RESULT_FOLDER.iterdir():
try:
if item.is_dir():
rmtree(item)
else:
item.unlink()
except Exception:
continue
print("⚠️ CLEAR_AT_RESTART=1 detected")
print("💥 Dropping all tables...")
db.drop_all()
print("🗄️ Recreating schema...")
db.create_all()
else:
if not tables:
print("🗄️ Creating database schema...")
db.create_all()
else:
print("🗄️ Database schema already exists")
print("📦 Filling IHDR static data...")
fill_ihdr_db()
print("✅ Database initialization complete")
RESULT_FOLDER.mkdir(parents=True, exist_ok=True)
if __name__ == "__main__":
main()
aperisolve/utils/png.py
+2 -1
View File
@@ -9,7 +9,7 @@ import struct
import zlib
from typing import Any, Optional
from ..app import app
from ..app import create_app
from ..models import IHDR
from .utils import int2hex, str2hex
@@ -241,6 +241,7 @@ class PNG:
crc_int = struct.unpack("!I", crc)[0]
# Database lookup
app = create_app()
with app.app_context():
matches = IHDR.query.filter_by(crc=crc_int).all()
aperisolve/sentry.py → aperisolve/utils/sentry.py
View File
aperisolve/wsgi.py → aperisolve/utils/wsgi.py
+2 -3
View File
@@ -3,7 +3,6 @@
# mypy: disable-error-code=unused-awaitable
"""Aperi'Solve WSGI runner."""
from .app import app as application
from aperisolve.app import create_app
if __name__ == "__main__":
application.run()
application = create_app()
aperisolve/workers.py
+4 -3
View File
@@ -9,7 +9,7 @@ from typing import Any
import sentry_sdk
from .sentry import initialize_sentry
from .utils.sentry import initialize_sentry
initialize_sentry()
@@ -24,9 +24,9 @@ from .analyzers.pngcheck import analyze_pngcheck
from .analyzers.steghide import analyze_steghide
from .analyzers.strings import analyze_strings
from .analyzers.zsteg import analyze_zsteg
from .app import app, db
from .app import create_app
from .config import RESULT_FOLDER
from .models import Image, Submission
from .models import Image, Submission, db
def analyze_image(submission_hash: str) -> None:
@@ -55,6 +55,7 @@ def analyze_image(submission_hash: str) -> None:
- Creates result directories and generates analysis output files
- Modifies the database session and commits changes
"""
app = create_app()
with app.app_context():
submission: Submission = Submission.query.get(submission_hash) # type: ignore
image = Image.query.get_or_404(submission.image_hash) # type: ignore
compose.dev.yml
+21 -10
View File
@@ -12,16 +12,14 @@ services:
depends_on:
postgres:
condition: service_healthy
initdb:
condition: service_completed_successfully
redis:
condition: service_started
worker:
condition: service_started
env_file:
- .env.example
environment:
- FLASK_DEBUG=true
- FLASK_ENV=development
- CLEAR_AT_RESTART=0 # Don't clear on Flask debug auto-reload
worker:
#image: ghcr.io/zeecka/aperisolve:latest
@@ -32,14 +30,27 @@ services:
volumes:
- ./aperisolve:/app/aperisolve
depends_on:
- redis
- postgres
postgres:
condition: service_healthy
initdb:
condition: service_completed_successfully
redis:
condition: service_started
env_file:
- .env.example
environment:
- FLASK_DEBUG=true
- FLASK_ENV=development
- CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing
initdb:
build: .
container_name: aperisolve-initdb
command: python -m aperisolve.utils.init_db
volumes:
- ./aperisolve:/app/aperisolve
depends_on:
postgres:
condition: service_healthy
env_file:
- .env.example
restart: "no"
redis:
image: redis:7
compose.yml
+27 -10
View File
@@ -1,9 +1,9 @@
services:
web:
image: ghcr.io/zeecka/aperisolve:latest
#build: .
#image: ghcr.io/zeecka/aperisolve:latest
build: .
container_name: aperisolve-web
restart: always
restart: no
ports:
- "5000:5000"
volumes:
@@ -11,6 +11,8 @@ services:
depends_on:
postgres:
condition: service_healthy
initdb:
condition: service_completed_successfully
redis:
condition: service_started
worker:
@@ -25,20 +27,22 @@ services:
start_period: 900s
worker:
image: ghcr.io/zeecka/aperisolve:latest
#build: .
#image: ghcr.io/zeecka/aperisolve:latest
build: .
container_name: aperisolve-worker
restart: always
restart: no
command: rq worker default --url redis://redis:6379/0
volumes:
- apresults:/app/aperisolve/results
depends_on:
- redis
- postgres
postgres:
condition: service_healthy
initdb:
condition: service_completed_successfully
redis:
condition: service_started
env_file:
- .env
environment:
- CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing
postgres:
image: postgres:18
@@ -54,6 +58,19 @@ services:
env_file:
- .env
initdb:
build: .
container_name: aperisolve-initdb
command: python -m aperisolve.utils.init_db
volumes:
- apresults:/app/aperisolve/results
depends_on:
postgres:
condition: service_healthy
env_file:
- .env
restart: "no"
redis:
image: redis:7
container_name: redis