Huge refactoring
This commit is contained in:
+1
-2
@@ -1,5 +1,5 @@
|
|||||||
# Application configuration
|
# Application configuration
|
||||||
MAX_CONTENT_LENGTH=1048576
|
MAX_CONTENT_LENGTH=3145728
|
||||||
MAX_PENDING_TIME=600
|
MAX_PENDING_TIME=600
|
||||||
MAX_STORE_TIME=259200
|
MAX_STORE_TIME=259200
|
||||||
CLEAR_AT_RESTART=1
|
CLEAR_AT_RESTART=1
|
||||||
@@ -7,7 +7,6 @@ SKIP_IHDR_FILL=0
|
|||||||
|
|
||||||
# Flask configuration
|
# Flask configuration
|
||||||
FLASK_DEBUG=0
|
FLASK_DEBUG=0
|
||||||
FLASK_ENV=development
|
|
||||||
|
|
||||||
# Redis configuration
|
# Redis configuration
|
||||||
RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0
|
RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0
|
||||||
|
|||||||
+1
-1
@@ -49,4 +49,4 @@ COPY aperisolve/ /app/aperisolve/
|
|||||||
RUN pip install --no-cache-dir -r /app/aperisolve/requirements.txt
|
RUN pip install --no-cache-dir -r /app/aperisolve/requirements.txt
|
||||||
|
|
||||||
# Commande de lancement
|
# Commande de lancement
|
||||||
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "--access-logfile", "-", "--error-logfile", "-", "--log-level", "info", "--capture-output", "aperisolve.wsgi:application"]
|
CMD ["gunicorn", "-w", "8", "-b", "0.0.0.0:5000", "--access-logfile", "-", "--error-logfile", "-", "--log-level", "info", "--capture-output", "aperisolve.utils.wsgi:application"]
|
||||||
|
|||||||
@@ -1,3 +1,6 @@
|
|||||||
# flake8: noqa: E203,E501,W503
|
# flake8: noqa: E203,E501,W503
|
||||||
# pylint: disable=C0413,W0718,R0903,R0801
|
# pylint: disable=C0413,W0718,R0903,R0801
|
||||||
# mypy: disable-error-code=unused-awaitable
|
# mypy: disable-error-code=unused-awaitable
|
||||||
|
from . import app, config, models, workers
|
||||||
|
|
||||||
|
__all__ = ["app", "config", "models", "workers"]
|
||||||
|
|||||||
@@ -13,8 +13,9 @@ from pathlib import Path
|
|||||||
from shutil import rmtree
|
from shutil import rmtree
|
||||||
from typing import Any, Optional, overload
|
from typing import Any, Optional, overload
|
||||||
|
|
||||||
|
from ..config import MAX_PENDING_TIME
|
||||||
|
|
||||||
_thread_lock = threading.Lock()
|
_thread_lock = threading.Lock()
|
||||||
MAX_PENDING_TIME = int(os.getenv("MAX_PENDING_TIME", "600")) # 10 minutes by default
|
|
||||||
|
|
||||||
|
|
||||||
class SubprocessAnalyzer(ABC):
|
class SubprocessAnalyzer(ABC):
|
||||||
|
|||||||
+14
-29
@@ -16,28 +16,23 @@ from flask import Flask, Response, abort, jsonify, render_template, request, sen
|
|||||||
from redis import Redis
|
from redis import Redis
|
||||||
from rq import Queue
|
from rq import Queue
|
||||||
|
|
||||||
from .sentry import initialize_sentry
|
from .utils.sentry import initialize_sentry
|
||||||
|
|
||||||
initialize_sentry()
|
initialize_sentry()
|
||||||
|
|
||||||
from .cleanup import cleanup_old_entries
|
|
||||||
from .config import IMAGE_EXTENSIONS, RESULT_FOLDER, WORKER_FILES
|
from .config import IMAGE_EXTENSIONS, RESULT_FOLDER, WORKER_FILES
|
||||||
from .models import Image, Submission, db, init_db
|
from .models import Image, Submission, cleanup_old_entries, db
|
||||||
|
|
||||||
app: Flask = Flask(__name__)
|
|
||||||
|
def create_app() -> Flask:
|
||||||
|
"""Create flask application with routes."""
|
||||||
|
app = Flask(__name__)
|
||||||
app.json.sort_keys = False # type: ignore
|
app.json.sort_keys = False # type: ignore
|
||||||
app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("DB_URI")
|
app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("DB_URI")
|
||||||
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
|
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
|
||||||
app.config["MAX_CONTENT_LENGTH"] = int(os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024))
|
app.config["MAX_CONTENT_LENGTH"] = int(os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024))
|
||||||
RESULT_FOLDER.mkdir(parents=True, exist_ok=True)
|
app.config["REDIS_QUEUE"] = Queue(connection=Redis(host="redis", port=6379))
|
||||||
|
|
||||||
db.init_app(app)
|
db.init_app(app)
|
||||||
with app.app_context():
|
|
||||||
init_db(app)
|
|
||||||
|
|
||||||
redis_conn = Redis(host="redis", port=6379)
|
|
||||||
queue = Queue(connection=redis_conn)
|
|
||||||
|
|
||||||
|
|
||||||
@app.errorhandler(413)
|
@app.errorhandler(413)
|
||||||
def too_large(_: Any) -> tuple[Response, int]:
|
def too_large(_: Any) -> tuple[Response, int]:
|
||||||
@@ -52,28 +47,21 @@ def too_large(_: Any) -> tuple[Response, int]:
|
|||||||
sentry_sdk.capture_message("Upload failed: 413 Payload Too Large", level="warning")
|
sentry_sdk.capture_message("Upload failed: 413 Payload Too Large", level="warning")
|
||||||
return jsonify({"error": "Image size exceeded"}), 413
|
return jsonify({"error": "Image size exceeded"}), 413
|
||||||
|
|
||||||
|
|
||||||
@app.errorhandler(404)
|
@app.errorhandler(404)
|
||||||
def not_found(_: Any) -> str:
|
def not_found(_: Any) -> str:
|
||||||
"""Error Handler for 404 not found."""
|
"""Error Handler for 404 not found."""
|
||||||
return render_template("error.html", message="Resource not found", statuscode=404)
|
return render_template("error.html", message="Resource not found", statuscode=404)
|
||||||
|
|
||||||
|
|
||||||
# Sentry automatically captures 500 errors
|
|
||||||
|
|
||||||
|
|
||||||
@app.route("/")
|
@app.route("/")
|
||||||
def index() -> str:
|
def index() -> str:
|
||||||
"""Render the main index page."""
|
"""Render the main index page."""
|
||||||
return render_template("index.html")
|
return render_template("index.html")
|
||||||
|
|
||||||
|
|
||||||
@app.route("/faq")
|
@app.route("/faq")
|
||||||
def faq() -> str:
|
def faq() -> str:
|
||||||
"""Render the FAQ page."""
|
"""Render the FAQ page."""
|
||||||
return render_template("faq.html")
|
return render_template("faq.html")
|
||||||
|
|
||||||
|
|
||||||
@app.route("/show")
|
@app.route("/show")
|
||||||
def show() -> str:
|
def show() -> str:
|
||||||
"""Show all active submissions."""
|
"""Show all active submissions."""
|
||||||
@@ -84,7 +72,6 @@ def show() -> str:
|
|||||||
images.append({"file": f"/image/{img.hash}", "link": f"/{last_sub.hash}"})
|
images.append({"file": f"/image/{img.hash}", "link": f"/{last_sub.hash}"})
|
||||||
return render_template("show.html", images=images)
|
return render_template("show.html", images=images)
|
||||||
|
|
||||||
|
|
||||||
@app.route("/<hash_val>", methods=["GET"])
|
@app.route("/<hash_val>", methods=["GET"])
|
||||||
def result_page(hash_val: str) -> str:
|
def result_page(hash_val: str) -> str:
|
||||||
"""Render the result page for a given submission hash."""
|
"""Render the result page for a given submission hash."""
|
||||||
@@ -93,7 +80,6 @@ def result_page(hash_val: str) -> str:
|
|||||||
|
|
||||||
return render_template("result.html", hash_val=hash_val)
|
return render_template("result.html", hash_val=hash_val)
|
||||||
|
|
||||||
|
|
||||||
@app.route("/upload", methods=["POST"])
|
@app.route("/upload", methods=["POST"])
|
||||||
def upload_image() -> tuple[Response, int]:
|
def upload_image() -> tuple[Response, int]:
|
||||||
"""Handle image upload and initiate analysis."""
|
"""Handle image upload and initiate analysis."""
|
||||||
@@ -186,19 +172,19 @@ def upload_image() -> tuple[Response, int]:
|
|||||||
db.session.commit() # pylint: disable=no-member
|
db.session.commit() # pylint: disable=no-member
|
||||||
|
|
||||||
# Start the analysis jobs
|
# Start the analysis jobs
|
||||||
queue.enqueue("aperisolve.workers.analyze_image", submission.hash, job_timeout=300)
|
app.config["REDIS_QUEUE"].enqueue(
|
||||||
|
"aperisolve.workers.analyze_image", submission.hash, job_timeout=300
|
||||||
|
)
|
||||||
|
|
||||||
# Return submission identifier
|
# Return submission identifier
|
||||||
return jsonify({"submission_hash": submission.hash}), 200
|
return jsonify({"submission_hash": submission.hash}), 200
|
||||||
|
|
||||||
|
|
||||||
@app.route("/status/<hash_val>", methods=["GET"])
|
@app.route("/status/<hash_val>", methods=["GET"])
|
||||||
def get_status(hash_val: str) -> Response:
|
def get_status(hash_val: str) -> Response:
|
||||||
"""Get the processing status of a submission."""
|
"""Get the processing status of a submission."""
|
||||||
submission = Submission.query.get_or_404(hash_val) # type: ignore
|
submission = Submission.query.get_or_404(hash_val) # type: ignore
|
||||||
return jsonify({"status": submission.status}) # type: ignore
|
return jsonify({"status": submission.status}) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
@app.route("/infos/<hash_val>", methods=["GET"])
|
@app.route("/infos/<hash_val>", methods=["GET"])
|
||||||
def get_infos(hash_val: str) -> Response:
|
def get_infos(hash_val: str) -> Response:
|
||||||
"""Get the metadata and information of a submission."""
|
"""Get the metadata and information of a submission."""
|
||||||
@@ -218,7 +204,6 @@ def get_infos(hash_val: str) -> Response:
|
|||||||
}
|
}
|
||||||
) # type: ignore
|
) # type: ignore
|
||||||
|
|
||||||
|
|
||||||
@app.route("/result/<hash_val>", methods=["GET"])
|
@app.route("/result/<hash_val>", methods=["GET"])
|
||||||
def get_result(hash_val: str) -> tuple[Response, int]:
|
def get_result(hash_val: str) -> tuple[Response, int]:
|
||||||
"""Get the analysis results of a submission."""
|
"""Get the analysis results of a submission."""
|
||||||
@@ -235,7 +220,6 @@ def get_result(hash_val: str) -> tuple[Response, int]:
|
|||||||
|
|
||||||
return jsonify({"results": results}), 200
|
return jsonify({"results": results}), 200
|
||||||
|
|
||||||
|
|
||||||
@app.route("/download/<hash_val>/<tool>")
|
@app.route("/download/<hash_val>/<tool>")
|
||||||
def download_output(hash_val: str, tool: str) -> Response:
|
def download_output(hash_val: str, tool: str) -> Response:
|
||||||
"""Download the output of a specific analyzer for a given submission hash."""
|
"""Download the output of a specific analyzer for a given submission hash."""
|
||||||
@@ -251,7 +235,6 @@ def download_output(hash_val: str, tool: str) -> Response:
|
|||||||
|
|
||||||
return send_file(output_file, as_attachment=True)
|
return send_file(output_file, as_attachment=True)
|
||||||
|
|
||||||
|
|
||||||
@app.route("/image/<img_name>")
|
@app.route("/image/<img_name>")
|
||||||
@app.route("/image/<hash_val>/<img_name>")
|
@app.route("/image/<hash_val>/<img_name>")
|
||||||
def get_image(hash_val: Optional[str] = None, img_name: Optional[str] = None) -> Response:
|
def get_image(hash_val: Optional[str] = None, img_name: Optional[str] = None) -> Response:
|
||||||
@@ -279,7 +262,9 @@ def get_image(hash_val: Optional[str] = None, img_name: Optional[str] = None) ->
|
|||||||
|
|
||||||
return send_file(output_file, as_attachment=True)
|
return send_file(output_file, as_attachment=True)
|
||||||
|
|
||||||
|
return app
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
debug_mode = os.environ.get("FLASK_DEBUG", "false").lower() == "true"
|
my_app = create_app()
|
||||||
app.run(host="0.0.0.0", port=5000, debug=debug_mode)
|
debug_mode = os.environ.get("FLASK_DEBUG", "0") == "1"
|
||||||
|
my_app.run(host="0.0.0.0", port=5000, debug=debug_mode)
|
||||||
|
|||||||
@@ -1,79 +0,0 @@
|
|||||||
# flake8: noqa: E203,E501,W503
|
|
||||||
# pylint: disable=C0413,W0718,R0903,R0801
|
|
||||||
# mypy: disable-error-code=unused-awaitable
|
|
||||||
"""Cleanup old entries in the database and file system."""
|
|
||||||
|
|
||||||
import shutil
|
|
||||||
import time
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from os import getenv
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from .models import Image, Submission, db
|
|
||||||
|
|
||||||
RESULT_FOLDER = Path("./results")
|
|
||||||
MAX_PENDING_TIME = int(getenv("MAX_PENDING_TIME", "600")) # 10 minutes by default
|
|
||||||
MAX_STORE_TIME = int(getenv("MAX_STORE_TIME", "259200")) # 3 days by default
|
|
||||||
|
|
||||||
|
|
||||||
def cleanup_old_entries() -> None:
|
|
||||||
"""
|
|
||||||
Clean up old and incomplete entries from the database and file system.
|
|
||||||
|
|
||||||
This function performs two main cleanup operations:
|
|
||||||
|
|
||||||
1. Submission cleanup:
|
|
||||||
- Deletes submissions with "pending" or "running" status that have exceeded
|
|
||||||
the maximum allowed processing time (MAX_PENDING_TIME).
|
|
||||||
- Deletes completed submissions ("done" status) that have missing or buggy
|
|
||||||
results (results.json file not found).
|
|
||||||
|
|
||||||
2. Image cleanup:
|
|
||||||
- Deletes images older than MAX_STORE_TIME along with all associated
|
|
||||||
submissions and their result folders.
|
|
||||||
- Deletes orphaned images (with no submissions) older than MAX_PENDING_TIME
|
|
||||||
and removes their associated result folders from the file system.
|
|
||||||
|
|
||||||
All database changes are committed at the end of the operation.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
None
|
|
||||||
"""
|
|
||||||
now = time.time()
|
|
||||||
for submission in Submission.query.all(): # type: ignore
|
|
||||||
if (
|
|
||||||
submission.status in ("pending", "running") # type: ignore
|
|
||||||
and now - submission.date > MAX_PENDING_TIME # type: ignore
|
|
||||||
):
|
|
||||||
# Processing took too long, delete
|
|
||||||
db.session.delete(submission) # pylint: disable=no-member
|
|
||||||
elif submission.status == "done": # type: ignore
|
|
||||||
# Search for buggy results, delete
|
|
||||||
result_path = RESULT_FOLDER / str(submission.image_hash) / str(submission.hash)
|
|
||||||
result_file = result_path / "results.json"
|
|
||||||
if not result_file.exists():
|
|
||||||
db.session.delete(submission) # pylint: disable=no-member
|
|
||||||
shutil.rmtree(result_path)
|
|
||||||
|
|
||||||
# Delete "old" images
|
|
||||||
for img in Image.query.all(): # type: ignore
|
|
||||||
if img.last_submission_date.tzinfo is None:
|
|
||||||
img_date = img.last_submission_date.replace(tzinfo=timezone.utc)
|
|
||||||
else:
|
|
||||||
img_date = img.last_submission_date
|
|
||||||
delay = datetime.now(timezone.utc) - img_date
|
|
||||||
img_fold = RESULT_FOLDER / img.hash
|
|
||||||
if delay.total_seconds() > MAX_STORE_TIME:
|
|
||||||
for s in img.submissions:
|
|
||||||
db.session.delete(s) # pylint: disable=no-member
|
|
||||||
if img_fold.exists():
|
|
||||||
shutil.rmtree(img_fold) # type: ignore
|
|
||||||
db.session.delete(img) # pylint: disable=no-member
|
|
||||||
|
|
||||||
# Delete Images with missing submission
|
|
||||||
if len(img.submissions) == 0 and delay.total_seconds() > MAX_PENDING_TIME:
|
|
||||||
if img_fold.exists():
|
|
||||||
shutil.rmtree(img_fold) # type: ignore
|
|
||||||
db.session.delete(img) # pylint: disable=no-member
|
|
||||||
|
|
||||||
db.session.commit() # pylint: disable=no-member
|
|
||||||
@@ -3,9 +3,13 @@
|
|||||||
# mypy: disable-error-code=unused-awaitable
|
# mypy: disable-error-code=unused-awaitable
|
||||||
"""Aperi'Solve configuration variables."""
|
"""Aperi'Solve configuration variables."""
|
||||||
|
|
||||||
|
from os import getenv
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
IMAGE_EXTENSIONS = [".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".tiff"]
|
IMAGE_EXTENSIONS = [".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".tiff"]
|
||||||
WORKER_FILES = ["binwalk", "foremost", "steghide", "zsteg", "image_resize", "openstego", "pcrt"]
|
WORKER_FILES = ["binwalk", "foremost", "steghide", "zsteg", "image_resize", "openstego", "pcrt"]
|
||||||
|
|
||||||
RESULT_FOLDER = Path(__file__).parent.resolve() / "results"
|
RESULT_FOLDER = Path(__file__).parent.resolve() / "results"
|
||||||
|
MAX_PENDING_TIME = int(getenv("MAX_PENDING_TIME", "600")) # 10 minutes by default
|
||||||
|
MAX_STORE_TIME = int(getenv("MAX_STORE_TIME", "259200")) # 3 days by default
|
||||||
|
CLEAR_AT_RESTART = int(getenv("CLEAR_AT_RESTART", "0"))
|
||||||
|
|||||||
+74
-77
@@ -1,18 +1,15 @@
|
|||||||
# flake8: noqa: E203,E501,W503
|
# flake8: noqa: E203,E501,W503
|
||||||
# pylint: disable=C0413,W0718,R0903,R0801
|
# pylint: disable=E1101,C0413,W0212,W0718,R0903,R0801
|
||||||
# mypy: disable-error-code=unused-awaitable
|
# mypy: disable-error-code=unused-awaitable
|
||||||
"""This module defines the database models for the Aperi'Solve application."""
|
"""This module defines the database models for the Aperi'Solve application."""
|
||||||
|
|
||||||
import itertools
|
import itertools
|
||||||
|
import shutil
|
||||||
import struct
|
import struct
|
||||||
import sys
|
import time
|
||||||
import zlib
|
import zlib
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from os import getenv
|
|
||||||
from pathlib import Path
|
|
||||||
from shutil import rmtree
|
|
||||||
|
|
||||||
from flask import Flask
|
|
||||||
from flask_sqlalchemy import SQLAlchemy
|
from flask_sqlalchemy import SQLAlchemy
|
||||||
from sqlalchemy import (
|
from sqlalchemy import (
|
||||||
BigInteger,
|
BigInteger,
|
||||||
@@ -25,6 +22,7 @@ from sqlalchemy import (
|
|||||||
String,
|
String,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
from aperisolve.config import MAX_PENDING_TIME, MAX_STORE_TIME, RESULT_FOLDER
|
||||||
from aperisolve.utils.utils import get_resolutions, get_valid_depth_color_pairs
|
from aperisolve.utils.utils import get_resolutions, get_valid_depth_color_pairs
|
||||||
|
|
||||||
db: SQLAlchemy = SQLAlchemy()
|
db: SQLAlchemy = SQLAlchemy()
|
||||||
@@ -117,9 +115,10 @@ def fill_ihdr_db() -> None:
|
|||||||
|
|
||||||
Computes CRC for each combination and stores parameters directly.
|
Computes CRC for each combination and stores parameters directly.
|
||||||
"""
|
"""
|
||||||
|
try:
|
||||||
# Skip if table already has entries
|
# Skip if table already has entries (check before doing any work)
|
||||||
if db.session.query(IHDR.iid).first() is not None:
|
count_query = db.session.query(IHDR.iid).limit(1)
|
||||||
|
if db.session.execute(count_query).first() is not None:
|
||||||
print("IHDR table already populated, skipping fill.")
|
print("IHDR table already populated, skipping fill.")
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -132,95 +131,93 @@ def fill_ihdr_db() -> None:
|
|||||||
bit_color_pairs,
|
bit_color_pairs,
|
||||||
interlace_methods,
|
interlace_methods,
|
||||||
)
|
)
|
||||||
batch = []
|
|
||||||
batch_size = 10000 # Increased from 5000
|
|
||||||
count = 0
|
count = 0
|
||||||
|
|
||||||
with db.session.no_autoflush:
|
with db.session.no_autoflush:
|
||||||
for (width, height), (bit_depth, color_type), interlace in combinations:
|
for (width, height), (bit_depth, color_type), interlace in combinations:
|
||||||
# Compute CRC directly
|
# Compute CRC directly
|
||||||
crc = IHDR.compute_crc(width, height, bit_depth, color_type, interlace)
|
crc = IHDR.compute_crc(width, height, bit_depth, color_type, interlace)
|
||||||
|
|
||||||
# Check if entry already exists
|
# Add new entry
|
||||||
exists = db.session.execute(
|
db.session.add(
|
||||||
db.select(IHDR.iid).where(
|
IHDR(
|
||||||
db.and_(
|
crc=crc,
|
||||||
IHDR.crc == crc,
|
width=width,
|
||||||
IHDR.width == width,
|
height=height,
|
||||||
IHDR.height == height,
|
bit_depth=bit_depth,
|
||||||
IHDR.bit_depth == bit_depth,
|
color_type=color_type,
|
||||||
IHDR.color_type == color_type,
|
interlace=interlace,
|
||||||
IHDR.interlace == interlace,
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
).first()
|
|
||||||
|
|
||||||
if exists:
|
|
||||||
continue
|
|
||||||
|
|
||||||
batch.append(
|
|
||||||
{
|
|
||||||
"crc": crc,
|
|
||||||
"width": width,
|
|
||||||
"height": height,
|
|
||||||
"bit_depth": bit_depth,
|
|
||||||
"color_type": color_type,
|
|
||||||
"interlace": interlace,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
count += 1
|
count += 1
|
||||||
if len(batch) >= batch_size:
|
if count % 5000 == 0:
|
||||||
db.session.bulk_insert_mappings(IHDR, batch)
|
|
||||||
db.session.commit()
|
|
||||||
print(f"Inserted {count} IHDR entries...")
|
print(f"Inserted {count} IHDR entries...")
|
||||||
batch = []
|
|
||||||
|
|
||||||
# Insert remaining entries
|
|
||||||
if batch:
|
|
||||||
db.session.bulk_insert_mappings(IHDR, batch)
|
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
|
|
||||||
print(f"Inserted {count} total IHDR entries.")
|
print(f"Inserted {count} total IHDR entries.")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error filling IHDR table: {e}")
|
||||||
|
db.session.rollback()
|
||||||
|
|
||||||
def init_db(app: Flask) -> None:
|
def cleanup_old_entries() -> None:
|
||||||
"""
|
"""
|
||||||
Initialize the database by creating tables and populating lookup data.
|
Clean up old and incomplete entries from the database and file system.
|
||||||
|
|
||||||
This function creates the database schema if it doesn't already exist,
|
This function performs two main cleanup operations:
|
||||||
and populates the CRC lookup table for PNG IHDR validation.
|
|
||||||
|
|
||||||
Args:
|
1. Submission cleanup:
|
||||||
app (Flask): The Flask application instance used to establish the application context
|
- Deletes submissions with "pending" or "running" status that have exceeded
|
||||||
for database operations.
|
the maximum allowed processing time (MAX_PENDING_TIME).
|
||||||
|
- Deletes completed submissions ("done" status) that have missing or buggy
|
||||||
|
results (results.json file not found).
|
||||||
|
|
||||||
|
2. Image cleanup:
|
||||||
|
- Deletes images older than MAX_STORE_TIME along with all associated
|
||||||
|
submissions and their result folders.
|
||||||
|
- Deletes orphaned images (with no submissions) older than MAX_PENDING_TIME
|
||||||
|
and removes their associated result folders from the file system.
|
||||||
|
|
||||||
|
All database changes are committed at the end of the operation.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
None
|
None
|
||||||
|
|
||||||
Side Effects:
|
|
||||||
- Creates all database tables if the "image" table doesn't exist
|
|
||||||
- Populates the IHDR CRC lookup table with initial data
|
|
||||||
- Prints status messages to console during initialization
|
|
||||||
"""
|
"""
|
||||||
# Detect if running as RQ worker by checking command line
|
now = time.time()
|
||||||
# Worker: /usr/local/bin/rq worker ...
|
for submission in Submission.query.all(): # type: ignore
|
||||||
# Web: /usr/local/bin/flask run ... or gunicorn/wsgi
|
if (
|
||||||
is_worker = any("rq" in arg and "flask" not in arg for arg in sys.argv[:2])
|
submission.status in ("pending", "running") # type: ignore
|
||||||
|
and now - submission.date > MAX_PENDING_TIME # type: ignore
|
||||||
|
):
|
||||||
|
# Processing took too long, delete
|
||||||
|
db.session.delete(submission) # pylint: disable=no-member
|
||||||
|
elif submission.status == "done": # type: ignore
|
||||||
|
# Search for buggy results, delete
|
||||||
|
result_path = RESULT_FOLDER / str(submission.image_hash) / str(submission.hash)
|
||||||
|
result_file = result_path / "results.json"
|
||||||
|
if not result_file.exists():
|
||||||
|
db.session.delete(submission) # pylint: disable=no-member
|
||||||
|
shutil.rmtree(result_path)
|
||||||
|
|
||||||
with app.app_context():
|
# Delete "old" images
|
||||||
# Workers should never clear the database
|
for img in Image.query.all(): # type: ignore
|
||||||
if not is_worker and getenv("CLEAR_AT_RESTART", "0") == "1":
|
if img.last_submission_date.tzinfo is None:
|
||||||
print("Clearing database and file system at restart...")
|
img_date = img.last_submission_date.replace(tzinfo=timezone.utc)
|
||||||
db.session.remove() # pylint: disable=no-member
|
|
||||||
db.drop_all()
|
|
||||||
rmtree(Path("./results"), ignore_errors=True) # Clear results folder
|
|
||||||
|
|
||||||
# Always create tables (idempotent - safe to call multiple times)
|
|
||||||
db.create_all()
|
|
||||||
print("Database structure created successfully.")
|
|
||||||
|
|
||||||
# Workers should never fill IHDR table
|
|
||||||
if is_worker:
|
|
||||||
print("Running as worker, skipping IHDR table fill.")
|
|
||||||
else:
|
else:
|
||||||
fill_ihdr_db()
|
img_date = img.last_submission_date
|
||||||
|
delay = datetime.now(timezone.utc) - img_date
|
||||||
|
img_fold = RESULT_FOLDER / img.hash
|
||||||
|
if delay.total_seconds() > MAX_STORE_TIME:
|
||||||
|
for s in img.submissions:
|
||||||
|
db.session.delete(s) # pylint: disable=no-member
|
||||||
|
if img_fold.exists():
|
||||||
|
shutil.rmtree(img_fold) # type: ignore
|
||||||
|
db.session.delete(img) # pylint: disable=no-member
|
||||||
|
|
||||||
|
# Delete Images with missing submission
|
||||||
|
if len(img.submissions) == 0 and delay.total_seconds() > MAX_PENDING_TIME:
|
||||||
|
if img_fold.exists():
|
||||||
|
shutil.rmtree(img_fold) # type: ignore
|
||||||
|
db.session.delete(img) # pylint: disable=no-member
|
||||||
|
|
||||||
|
db.session.commit() # pylint: disable=no-member
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
"""
|
||||||
|
Database initialization script.
|
||||||
|
This script is meant to be run ONCE at deploy time, never from Gunicorn or runtime code.
|
||||||
|
If CLEAR_AT_RESTART is set, database will be reset at launch time.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from shutil import rmtree
|
||||||
|
|
||||||
|
from sqlalchemy import inspect
|
||||||
|
|
||||||
|
from ..app import create_app
|
||||||
|
from ..config import RESULT_FOLDER
|
||||||
|
from ..models import db, fill_ihdr_db
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
"""Database initialization main function."""
|
||||||
|
reset = os.getenv("CLEAR_AT_RESTART") == "1"
|
||||||
|
|
||||||
|
app = create_app()
|
||||||
|
|
||||||
|
with app.app_context():
|
||||||
|
inspector = inspect(db.engine)
|
||||||
|
tables = inspector.get_table_names()
|
||||||
|
|
||||||
|
if reset:
|
||||||
|
|
||||||
|
# Delete result folder content
|
||||||
|
if RESULT_FOLDER.exists():
|
||||||
|
for item in RESULT_FOLDER.iterdir():
|
||||||
|
try:
|
||||||
|
if item.is_dir():
|
||||||
|
rmtree(item)
|
||||||
|
else:
|
||||||
|
item.unlink()
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
|
||||||
|
print("⚠️ CLEAR_AT_RESTART=1 detected")
|
||||||
|
print("💥 Dropping all tables...")
|
||||||
|
db.drop_all()
|
||||||
|
print("🗄️ Recreating schema...")
|
||||||
|
db.create_all()
|
||||||
|
|
||||||
|
else:
|
||||||
|
if not tables:
|
||||||
|
print("🗄️ Creating database schema...")
|
||||||
|
db.create_all()
|
||||||
|
else:
|
||||||
|
print("🗄️ Database schema already exists")
|
||||||
|
|
||||||
|
print("📦 Filling IHDR static data...")
|
||||||
|
fill_ihdr_db()
|
||||||
|
|
||||||
|
print("✅ Database initialization complete")
|
||||||
|
|
||||||
|
RESULT_FOLDER.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -9,7 +9,7 @@ import struct
|
|||||||
import zlib
|
import zlib
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
from ..app import app
|
from ..app import create_app
|
||||||
from ..models import IHDR
|
from ..models import IHDR
|
||||||
from .utils import int2hex, str2hex
|
from .utils import int2hex, str2hex
|
||||||
|
|
||||||
@@ -241,6 +241,7 @@ class PNG:
|
|||||||
crc_int = struct.unpack("!I", crc)[0]
|
crc_int = struct.unpack("!I", crc)[0]
|
||||||
|
|
||||||
# Database lookup
|
# Database lookup
|
||||||
|
app = create_app()
|
||||||
with app.app_context():
|
with app.app_context():
|
||||||
matches = IHDR.query.filter_by(crc=crc_int).all()
|
matches = IHDR.query.filter_by(crc=crc_int).all()
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,6 @@
|
|||||||
# mypy: disable-error-code=unused-awaitable
|
# mypy: disable-error-code=unused-awaitable
|
||||||
"""Aperi'Solve WSGI runner."""
|
"""Aperi'Solve WSGI runner."""
|
||||||
|
|
||||||
from .app import app as application
|
from aperisolve.app import create_app
|
||||||
|
|
||||||
if __name__ == "__main__":
|
application = create_app()
|
||||||
application.run()
|
|
||||||
@@ -9,7 +9,7 @@ from typing import Any
|
|||||||
|
|
||||||
import sentry_sdk
|
import sentry_sdk
|
||||||
|
|
||||||
from .sentry import initialize_sentry
|
from .utils.sentry import initialize_sentry
|
||||||
|
|
||||||
initialize_sentry()
|
initialize_sentry()
|
||||||
|
|
||||||
@@ -24,9 +24,9 @@ from .analyzers.pngcheck import analyze_pngcheck
|
|||||||
from .analyzers.steghide import analyze_steghide
|
from .analyzers.steghide import analyze_steghide
|
||||||
from .analyzers.strings import analyze_strings
|
from .analyzers.strings import analyze_strings
|
||||||
from .analyzers.zsteg import analyze_zsteg
|
from .analyzers.zsteg import analyze_zsteg
|
||||||
from .app import app, db
|
from .app import create_app
|
||||||
from .config import RESULT_FOLDER
|
from .config import RESULT_FOLDER
|
||||||
from .models import Image, Submission
|
from .models import Image, Submission, db
|
||||||
|
|
||||||
|
|
||||||
def analyze_image(submission_hash: str) -> None:
|
def analyze_image(submission_hash: str) -> None:
|
||||||
@@ -55,6 +55,7 @@ def analyze_image(submission_hash: str) -> None:
|
|||||||
- Creates result directories and generates analysis output files
|
- Creates result directories and generates analysis output files
|
||||||
- Modifies the database session and commits changes
|
- Modifies the database session and commits changes
|
||||||
"""
|
"""
|
||||||
|
app = create_app()
|
||||||
with app.app_context():
|
with app.app_context():
|
||||||
submission: Submission = Submission.query.get(submission_hash) # type: ignore
|
submission: Submission = Submission.query.get(submission_hash) # type: ignore
|
||||||
image = Image.query.get_or_404(submission.image_hash) # type: ignore
|
image = Image.query.get_or_404(submission.image_hash) # type: ignore
|
||||||
|
|||||||
+21
-10
@@ -12,16 +12,14 @@ services:
|
|||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
initdb:
|
||||||
|
condition: service_completed_successfully
|
||||||
redis:
|
redis:
|
||||||
condition: service_started
|
condition: service_started
|
||||||
worker:
|
worker:
|
||||||
condition: service_started
|
condition: service_started
|
||||||
env_file:
|
env_file:
|
||||||
- .env.example
|
- .env.example
|
||||||
environment:
|
|
||||||
- FLASK_DEBUG=true
|
|
||||||
- FLASK_ENV=development
|
|
||||||
- CLEAR_AT_RESTART=0 # Don't clear on Flask debug auto-reload
|
|
||||||
|
|
||||||
worker:
|
worker:
|
||||||
#image: ghcr.io/zeecka/aperisolve:latest
|
#image: ghcr.io/zeecka/aperisolve:latest
|
||||||
@@ -32,14 +30,27 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- ./aperisolve:/app/aperisolve
|
- ./aperisolve:/app/aperisolve
|
||||||
depends_on:
|
depends_on:
|
||||||
- redis
|
postgres:
|
||||||
- postgres
|
condition: service_healthy
|
||||||
|
initdb:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
redis:
|
||||||
|
condition: service_started
|
||||||
env_file:
|
env_file:
|
||||||
- .env.example
|
- .env.example
|
||||||
environment:
|
|
||||||
- FLASK_DEBUG=true
|
initdb:
|
||||||
- FLASK_ENV=development
|
build: .
|
||||||
- CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing
|
container_name: aperisolve-initdb
|
||||||
|
command: python -m aperisolve.utils.init_db
|
||||||
|
volumes:
|
||||||
|
- ./aperisolve:/app/aperisolve
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
env_file:
|
||||||
|
- .env.example
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
image: redis:7
|
image: redis:7
|
||||||
|
|||||||
+27
-10
@@ -1,9 +1,9 @@
|
|||||||
services:
|
services:
|
||||||
web:
|
web:
|
||||||
image: ghcr.io/zeecka/aperisolve:latest
|
#image: ghcr.io/zeecka/aperisolve:latest
|
||||||
#build: .
|
build: .
|
||||||
container_name: aperisolve-web
|
container_name: aperisolve-web
|
||||||
restart: always
|
restart: no
|
||||||
ports:
|
ports:
|
||||||
- "5000:5000"
|
- "5000:5000"
|
||||||
volumes:
|
volumes:
|
||||||
@@ -11,6 +11,8 @@ services:
|
|||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
initdb:
|
||||||
|
condition: service_completed_successfully
|
||||||
redis:
|
redis:
|
||||||
condition: service_started
|
condition: service_started
|
||||||
worker:
|
worker:
|
||||||
@@ -25,20 +27,22 @@ services:
|
|||||||
start_period: 900s
|
start_period: 900s
|
||||||
|
|
||||||
worker:
|
worker:
|
||||||
image: ghcr.io/zeecka/aperisolve:latest
|
#image: ghcr.io/zeecka/aperisolve:latest
|
||||||
#build: .
|
build: .
|
||||||
container_name: aperisolve-worker
|
container_name: aperisolve-worker
|
||||||
restart: always
|
restart: no
|
||||||
command: rq worker default --url redis://redis:6379/0
|
command: rq worker default --url redis://redis:6379/0
|
||||||
volumes:
|
volumes:
|
||||||
- apresults:/app/aperisolve/results
|
- apresults:/app/aperisolve/results
|
||||||
depends_on:
|
depends_on:
|
||||||
- redis
|
postgres:
|
||||||
- postgres
|
condition: service_healthy
|
||||||
|
initdb:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
redis:
|
||||||
|
condition: service_started
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
environment:
|
|
||||||
- CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing
|
|
||||||
|
|
||||||
postgres:
|
postgres:
|
||||||
image: postgres:18
|
image: postgres:18
|
||||||
@@ -54,6 +58,19 @@ services:
|
|||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
|
|
||||||
|
initdb:
|
||||||
|
build: .
|
||||||
|
container_name: aperisolve-initdb
|
||||||
|
command: python -m aperisolve.utils.init_db
|
||||||
|
volumes:
|
||||||
|
- apresults:/app/aperisolve/results
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
image: redis:7
|
image: redis:7
|
||||||
container_name: redis
|
container_name: redis
|
||||||
|
|||||||
Reference in New Issue
Block a user