From f83e6e571ed230295e21039d245bf24d95e8bae2 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 04:41:57 +0100 Subject: [PATCH 01/14] Fix 110 --- README.md | 27 +++++++++++++++++++++++---- compose.dev.yml | 18 +++++++++++------- compose.yml | 22 ++++++++++++---------- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 929dbc2..115dbae 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Support Aperi'Solve: Aperi'Solve is an open-source steganalysis web platform that performs automated analysis on images to detect and extract hidden data using common steganography tools and techniques. Table of contents + - [Key features](#key-features) - [Quick start (Docker)](#quick-start-docker) - [Development](#development) @@ -59,6 +60,7 @@ In case you want to host your own version of https://www.aperisolve.com/. Recommended: Docker + Docker Compose. Clone and start the full stack (production-like): + ```bash git clone https://github.com/Zeecka/AperiSolve.git cd AperiSolve @@ -68,13 +70,16 @@ docker compose up -d Default: http://localhost:5000/ ## Development + Development compose: + ```bash # development environment (hot reload and local volumes) docker compose -f compose.dev.yml up --build ``` Useful commands: + ```bash # Stop and remove containers, networks and volumes (results are stored in a volume) docker compose down -v @@ -84,21 +89,26 @@ docker exec -it aperisolve-web bash # Enter Postgres shell (from host) docker exec -it postgres psql -U aperiuser -d aperisolve + +# Backup all uploaded files +docker cp -r aperisolve-web:/aperisolve/results /path/to/backup/location + +# Backup a single uploaded file +docker cp aperisolve-web:/aperisolve/results/filename.ext /path/to/backup/filename.ext ``` > [!WARNING] -> If switching between dev and production compose files, remove the `results` directory or mounted volume to avoid conflicts: -> ```bash -> rm -rf aperisolve/results -> ``` +> Uploaded files are stored in a tmpfs. This means that they are stored in memory and will be lost if the server is restarted. Ensure that you back up any important files before the server is stopped or restarted. ### Adding a new analyzer Adding a custom analyzer is straightforward: 1. Create your analyzer file: + - Copy `aperisolve/analyzers/template_analyzer.py` -> `aperisolve/analyzers/myanalyzer.py` - Implement function signature similar to: + ```python # aperisolve/analyzers/myanalyzer.py def analyze_myanalyzer(image_path: str, results_dir: str) -> dict: @@ -111,7 +121,9 @@ Adding a custom analyzer is straightforward: ``` 2. Register the analyzer in the worker pipeline: + - Edit `aperisolve/workers.py`: + ```python # import from .analyzers.myanalyzer import analyze_myanalyzer @@ -126,23 +138,27 @@ Adding a custom analyzer is straightforward: ``` 3. Add your analyzer to the UI order: + - Edit `aperisolve/static/js/aperisolve.js` and append `myanalyzer` to `TOOL_ORDER` so it appears in the frontend. 4. Test locally: run the worker and submit jobs to ensure outputs are produced and displayed. > [!TIP] +> > - Keep analyzers idempotent and write outputs to the provided `results_dir`. > - Return structured JSON so the frontend can render links/downloads automatically. ## Configuration & environment variables Typical services: + - Web app (Flask) - Workers (Python) - Redis (RQ) - PostgreSQL Main environment variables (examples): + - DATABASE_URL=postgresql://aperiuser:password@postgres:5432/aperisolve - REDIS_URL=redis://redis:6379/0 - SECRET_KEY=change_me @@ -185,12 +201,14 @@ This separation keeps heavy tools (binwalk, foremost, zsteg, etc.) isolated and ## Contributing Contributions are welcome! Please: + 1. Fork the repository 2. Create a feature branch 3. Open a pull request describing your change > [!IMPORTANT] > Please follow the code style and run linters before submitting. The project adheres to: +> > - Black > - Flake8 (ignoring E203, E501, W503) > - Pylint (ignoring W0718, R0903, R0801) @@ -206,4 +224,5 @@ CI will run these checks on each PR. ## Credits Acknowledgements: + - Thanks to contributors and the open-source community for the tools integrated ([zsteg](https://github.com/zed-0xff/zsteg), [steghide](https://steghide.sourceforge.net/), [outguess](https://www.rbcafe.com/software/outguess/), [openstego](https://www.openstego.com/), [exiftool](https://exiftool.org/), [binwalk](https://github.com/ReFirmLabs/binwalk), [foremost](https://foremost.sourceforge.net/), [pngcheck](https://www.libpng.org/pub/png/apps/pngcheck.html), [strings](https://pubs.opengroup.org/onlinepubs/9799919799/utilities/strings.html), ...). diff --git a/compose.dev.yml b/compose.dev.yml index e23163d..ce877d3 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -9,6 +9,9 @@ services: - "5000:5000" volumes: - ./aperisolve:/aperisolve + # Commented tmpfs for development to allow persistent results during code changes + tmpfs: + - /aperisolve/results depends_on: postgres: condition: service_healthy @@ -30,6 +33,9 @@ services: command: rq worker default --url redis://redis:6379/0 volumes: - ./aperisolve:/aperisolve + # Commented tmpfs for development to allow persistent results during code changes + tmpfs: + - /aperisolve/results depends_on: - redis - postgres @@ -40,18 +46,19 @@ services: - FLASK_ENV=development redis: - image: redis:7 + image: redis:8 container_name: redis restart: always env_file: - .env postgres: - image: postgres:14 + image: postgres:18 container_name: postgres restart: always - volumes: - - pgdata:/var/lib/postgresql/data + # Commented tmpfs for development to allow persistent results during code changes + tmpfs: + - /var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s @@ -71,6 +78,3 @@ services: - redis env_file: - .env - -volumes: - pgdata: \ No newline at end of file diff --git a/compose.yml b/compose.yml index 777fef2..612016d 100644 --- a/compose.yml +++ b/compose.yml @@ -6,8 +6,8 @@ services: restart: always ports: - "5000:5000" - volumes: - - apresults:/aperisolve/results + tmpfs: + - /aperisolve/results depends_on: postgres: condition: service_healthy @@ -17,6 +17,12 @@ services: condition: service_started env_file: - .env + healthcheck: + test: ["CMD-SHELL", "dd if=/dev/urandom of=/tmp/healthcheck.png bs=1 count=1000000 && curl -X POST -F 'image=@/tmp/healthcheck.png' http://localhost:5000/upload -s -o /dev/null -w '%{http_code}' | grep -q 200"] + interval: 3600s + timeout: 30s + retries: 3 + start_period: 900s worker: image: ghcr.io/zeecka/aperisolve:latest @@ -24,8 +30,8 @@ services: container_name: aperisolve-worker restart: always command: rq worker default --url redis://redis:6379/0 - volumes: - - apresults:/aperisolve/results + tmpfs: + - /aperisolve/results depends_on: - redis - postgres @@ -36,8 +42,8 @@ services: image: postgres:14 container_name: postgres restart: always - volumes: - - pgdata:/var/lib/postgresql/data + tmpfs: + - /var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s @@ -63,7 +69,3 @@ services: - redis env_file: - .env - -volumes: - apresults: - pgdata: From 2f50a23b8ccbcd35b09a0699708cbd2742178e63 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 04:44:14 +0100 Subject: [PATCH 02/14] change readme roadmap --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 115dbae..48fa04c 100644 --- a/README.md +++ b/README.md @@ -187,11 +187,7 @@ This separation keeps heavy tools (binwalk, foremost, zsteg, etc.) isolated and ## Roadmap -- [ ] **[Bug]** Error 500 when file already exist on disc - [ ] **[Bug]** Duplicate key value violates unique constraint "submission_pkey" -- [ ] **[Bug]** Binwalk Extractor error -- [ ] **[Feature]** Implement healthcheck and auto reboot -- [ ] **[Feature]** Implement Sentry for better error handling - [ ] **[Feature]** Zsteg: full extraction (--all) and download of discovered files (mp3, etc.) - [ ] **[Feature]** Mobile-friendly UI / test on mobile - [ ] **[Feature]** i18n (internationalization) From 29cdeaea4c2e150be67d433c38e8c35410f62be8 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 04:45:07 +0100 Subject: [PATCH 03/14] format, black --- aperisolve/app.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index 3b1ee5d..295faaa 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -43,10 +43,13 @@ queue = Queue(connection=redis_conn) @app.errorhandler(413) def too_large(_: Any) -> tuple[Response, int]: """Error Handler for Max File Size.""" - sentry_sdk.set_context("upload", { - "content_length": request.content_length, - "max_allowed": app.config["MAX_CONTENT_LENGTH"], - }) + sentry_sdk.set_context( + "upload", + { + "content_length": request.content_length, + "max_allowed": app.config["MAX_CONTENT_LENGTH"], + }, + ) sentry_sdk.capture_message("Upload failed: 413 Payload Too Large", level="warning") return jsonify({"error": "Image size exceeded"}), 413 @@ -56,8 +59,10 @@ def not_found(_: Any) -> str: """Error Handler for 404 not found.""" return render_template("error.html", message="Resource not found", statuscode=404) + # Sentry automatically captures 500 errors + @app.route("/") def index() -> str: """Render the main index page.""" @@ -127,9 +132,7 @@ def upload_image() -> tuple[Response, int]: submission_path = RESULT_FOLDER / img_hash / submission_hash # Check if hash is present on the DB - submission: Submission = Submission.query.filter_by( - hash=submission_hash - ).first() + submission: Submission = Submission.query.filter_by(hash=submission_hash).first() # Only return if BOTH the file exists AND the DB entry exists if submission_path.exists() and submission is not None: @@ -145,9 +148,7 @@ def upload_image() -> tuple[Response, int]: f.write(image_data) # Try to find image in DB by hash - sub_img = Image.query.filter_by( - hash=img_hash - ).first() + sub_img = Image.query.filter_by(hash=img_hash).first() # If DB entry is missing (even if file exists on disk), create it! if sub_img is None: From fc02dcab80844b620de2edbde1bdacd544dda584 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 04:53:19 +0100 Subject: [PATCH 04/14] bump postgres_14 --- compose.dev.yml | 2 +- compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose.dev.yml b/compose.dev.yml index ce877d3..df6e5c2 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -58,7 +58,7 @@ services: restart: always # Commented tmpfs for development to allow persistent results during code changes tmpfs: - - /var/lib/postgresql/data + - /var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s diff --git a/compose.yml b/compose.yml index 612016d..1191cae 100644 --- a/compose.yml +++ b/compose.yml @@ -43,7 +43,7 @@ services: container_name: postgres restart: always tmpfs: - - /var/lib/postgresql/data + - /var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s From ddce01ec8d3a49c5c106eb0eae62526f673c18ce Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 05:28:58 +0100 Subject: [PATCH 05/14] change docker entrypoint to clear content --- Dockerfile | 2 +- aperisolve/static/js/aperisolve.js | 4 +++- compose.dev.yml | 25 +++++++++++++------------ compose.yml | 11 ++++++++--- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1c0ce92..cff09e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,4 +42,4 @@ RUN apt-get remove -y wget && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "--access-logfile", "-", "--error-logfile", "-", "--log-level", "info", "--capture-output", "aperisolve.wsgi:application"] +CMD ["sh", "-c", "rm -rf /aperisolve/results/* && gunicorn -w 4 -b 0.0.0.0:5000 --access-logfile - --error-logfile - --log-level info --capture-output aperisolve.wsgi:application"] diff --git a/aperisolve/static/js/aperisolve.js b/aperisolve/static/js/aperisolve.js index 0090c17..0294ead 100644 --- a/aperisolve/static/js/aperisolve.js +++ b/aperisolve/static/js/aperisolve.js @@ -486,7 +486,9 @@ if (browseBtn) { } catch (e) { showDanger("❌ Invalid server response.", true); } - } else { + } else if (xhr.status == 413) { + showDanger("❌ File too large. Please upload a smaller file.", true); + }else { showDanger(`❌ HTTP error ${xhr.status}`, true); } progressBar.style.width = "100%"; diff --git a/compose.dev.yml b/compose.dev.yml index df6e5c2..9df9dea 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -4,14 +4,13 @@ services: build: . container_name: aperisolve-web restart: always - command: ["flask", "--app", "aperisolve.app", "run", "--host=0.0.0.0", "--port=5000"] + command: ["/bin/sh", "-c", "rm -rf /aperisolve/results/* && flask --app aperisolve.app run --host=0.0.0.0 --port=5000"] + #command: ["/bin/sh", "-c", "flask --app aperisolve.app run --host=0.0.0.0 --port=5000"] ports: - "5000:5000" volumes: - ./aperisolve:/aperisolve - # Commented tmpfs for development to allow persistent results during code changes - tmpfs: - - /aperisolve/results + - apresults:/aperisolve/results # Shared named volume (renamed) depends_on: postgres: condition: service_healthy @@ -30,12 +29,11 @@ services: build: . container_name: aperisolve-worker restart: always - command: rq worker default --url redis://redis:6379/0 + command: ["/bin/sh", "-c", "rm -rf /aperisolve/results/* && rq worker default --url redis://redis:6379/0"] + #command: ["/bin/sh", "-c", "rq worker default --url redis://redis:6379/0"] volumes: - ./aperisolve:/aperisolve - # Commented tmpfs for development to allow persistent results during code changes - tmpfs: - - /aperisolve/results + - apresults:/aperisolve/results # Shared named volume (renamed) depends_on: - redis - postgres @@ -56,9 +54,9 @@ services: image: postgres:18 container_name: postgres restart: always - # Commented tmpfs for development to allow persistent results during code changes - tmpfs: - - /var/lib/postgresql + entrypoint: ["/bin/sh", "-c", "rm -rf /var/lib/postgresql/* && docker-entrypoint.sh postgres"] + volumes: + - pgdata:/var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s @@ -67,7 +65,6 @@ services: env_file: - .env - rqdashboard: image: eoranged/rq-dashboard container_name: rq-dashboard @@ -78,3 +75,7 @@ services: - redis env_file: - .env + +volumes: + apresults: + pgdata: diff --git a/compose.yml b/compose.yml index 1191cae..afa902f 100644 --- a/compose.yml +++ b/compose.yml @@ -39,11 +39,12 @@ services: - .env postgres: - image: postgres:14 + image: postgres:18 container_name: postgres restart: always - tmpfs: - - /var/lib/postgresql + entrypoint: ["/bin/sh", "-c", "rm -rf /var/lib/postgresql/* && docker-entrypoint.sh postgres"] + volumes: + - pgdata:/var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s @@ -69,3 +70,7 @@ services: - redis env_file: - .env + +volumes: + apresults: + pgdata: From 9c37efb403915efa81b5ca42f454b19e7d1b96f8 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 15:58:51 +0100 Subject: [PATCH 06/14] opti sentry --- aperisolve/sentry.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/aperisolve/sentry.py b/aperisolve/sentry.py index bb87683..e0677a4 100644 --- a/aperisolve/sentry.py +++ b/aperisolve/sentry.py @@ -12,35 +12,31 @@ from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration from sentry_sdk.integrations.threading import ThreadingIntegration SENTRY_DSN = os.environ.get("SENTRY_DSN") -ENVIRONMENT = os.environ.get("ENVIRONMENT", "development") - +SENTRY_ENVIRONMENT = os.environ.get("SENTRY_ENVIRONMENT", "development") +SENTRY_TRACES_SAMPLE_RATE = float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")) +SENTRY_PROFILES_SAMPLE_RATE = float(os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1")) +SENTRY_RELEASE = os.environ.get("SENTRY_RELEASE", "1.0.0") def initialize_sentry() -> None: """Initialize Sentry SDK once, safely.""" # No DSN → do nothing - dsn = os.environ.get("SENTRY_DSN") - if not dsn: - return - # Prevent double initialization (Flask + RQ, CLI, tests, etc.) - if sentry_sdk.Hub.current.client is not None: + if not SENTRY_DSN or sentry_sdk.Hub.current.client is not None: return sentry_sdk.init( - dsn=dsn, + dsn=SENTRY_DSN, integrations=[ FlaskIntegration(), SqlalchemyIntegration(), RqIntegration(), ThreadingIntegration(propagate_hub=True), ], - traces_sample_rate=float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")), - profiles_sample_rate=float( - os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1") - ), - environment=os.environ.get("ENVIRONMENT", "development"), - release=os.environ.get("SENTRY_RELEASE", "1.0.0"), + traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE, + profiles_sample_rate=SENTRY_PROFILES_SAMPLE_RATE, + environment=SENTRY_ENVIRONMENT, + release=SENTRY_RELEASE, send_default_pii=False, enable_tracing=True, ) From 58edd548355d0428f6a8ec077f64e49801cbb4eb Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 15:59:13 +0100 Subject: [PATCH 07/14] remove cleanup in compose and docker --- Dockerfile | 2 +- compose.dev.yml | 9 +++------ compose.yml | 9 ++++----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index cff09e7..1c0ce92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,4 +42,4 @@ RUN apt-get remove -y wget && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -CMD ["sh", "-c", "rm -rf /aperisolve/results/* && gunicorn -w 4 -b 0.0.0.0:5000 --access-logfile - --error-logfile - --log-level info --capture-output aperisolve.wsgi:application"] +CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "--access-logfile", "-", "--error-logfile", "-", "--log-level", "info", "--capture-output", "aperisolve.wsgi:application"] diff --git a/compose.dev.yml b/compose.dev.yml index 9df9dea..c10943f 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -4,13 +4,12 @@ services: build: . container_name: aperisolve-web restart: always - command: ["/bin/sh", "-c", "rm -rf /aperisolve/results/* && flask --app aperisolve.app run --host=0.0.0.0 --port=5000"] - #command: ["/bin/sh", "-c", "flask --app aperisolve.app run --host=0.0.0.0 --port=5000"] + command: ["/bin/sh", "-c", "flask --app aperisolve.app run --host=0.0.0.0 --port=5000"] ports: - "5000:5000" volumes: - ./aperisolve:/aperisolve - - apresults:/aperisolve/results # Shared named volume (renamed) + - apresults:/aperisolve/results depends_on: postgres: condition: service_healthy @@ -29,8 +28,7 @@ services: build: . container_name: aperisolve-worker restart: always - command: ["/bin/sh", "-c", "rm -rf /aperisolve/results/* && rq worker default --url redis://redis:6379/0"] - #command: ["/bin/sh", "-c", "rq worker default --url redis://redis:6379/0"] + command: ["/bin/sh", "-c", "rq worker default --url redis://redis:6379/0"] volumes: - ./aperisolve:/aperisolve - apresults:/aperisolve/results # Shared named volume (renamed) @@ -54,7 +52,6 @@ services: image: postgres:18 container_name: postgres restart: always - entrypoint: ["/bin/sh", "-c", "rm -rf /var/lib/postgresql/* && docker-entrypoint.sh postgres"] volumes: - pgdata:/var/lib/postgresql healthcheck: diff --git a/compose.yml b/compose.yml index afa902f..2afcb73 100644 --- a/compose.yml +++ b/compose.yml @@ -6,8 +6,8 @@ services: restart: always ports: - "5000:5000" - tmpfs: - - /aperisolve/results + volumes: + - apresults:/aperisolve/results depends_on: postgres: condition: service_healthy @@ -30,8 +30,8 @@ services: container_name: aperisolve-worker restart: always command: rq worker default --url redis://redis:6379/0 - tmpfs: - - /aperisolve/results + volumes: + - apresults:/aperisolve/results depends_on: - redis - postgres @@ -42,7 +42,6 @@ services: image: postgres:18 container_name: postgres restart: always - entrypoint: ["/bin/sh", "-c", "rm -rf /var/lib/postgresql/* && docker-entrypoint.sh postgres"] volumes: - pgdata:/var/lib/postgresql healthcheck: From 3a9cf21c24341c7e331b3bfae0aa25f44195f629 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 16:00:43 +0100 Subject: [PATCH 08/14] fix sentry env and add comments --- .env | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 4116ee9..549b813 100644 --- a/.env +++ b/.env @@ -1,15 +1,24 @@ +# Application configuration MAX_CONTENT_LENGTH=1048576 MAX_PENDING_TIME=600 MAX_STORE_TIME=259200 + +# Flask configuration FLASK_DEBUG=0 FLASK_ENV=production -DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve + +# Redis configuration RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0 + +# Database configuration POSTGRES_DB=aperisolve POSTGRES_USER=aperiuser POSTGRES_PASSWORD=aperipass +DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve + +# Sentry configuration SENTRY_DSN=https://your-dsn@sentry.io/project-id SENTRY_TRACES_SAMPLE_RATE=0.1 SENTRY_PROFILES_SAMPLE_RATE=0.1 SENTRY_RELEASE=1.0.0 -ENVIRONMENT=production \ No newline at end of file +SENTRY_ENVIRONMENT=production From 5c5e63dbea949bd43758b8ada040f67a98ce2cb5 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 16:25:54 +0100 Subject: [PATCH 09/14] SKIP IHDR and Clear at restart --- .env | 2 ++ aperisolve/models.py | 18 ++++++++++++++---- compose.dev.yml | 1 + 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 549b813..366a17d 100644 --- a/.env +++ b/.env @@ -2,6 +2,8 @@ MAX_CONTENT_LENGTH=1048576 MAX_PENDING_TIME=600 MAX_STORE_TIME=259200 +CLEAR_AT_RESTART=1 +SKIP_IHDR_FILL=1 # Flask configuration FLASK_DEBUG=0 diff --git a/aperisolve/models.py b/aperisolve/models.py index b064689..6b300f8 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -5,6 +5,9 @@ import itertools from datetime import datetime, timezone +from os import getenv +from pathlib import Path +from shutil import rmtree from flask import Flask from flask_sqlalchemy import SQLAlchemy @@ -128,9 +131,16 @@ def init_db(app: Flask) -> None: """ with app.app_context(): if not db.engine.dialect.has_table(db.engine.connect(), "image"): - print("Creating database...") + print(f"CLEAR_AT_RESTART={getenv('CLEAR_AT_RESTART', '0')}") + if getenv("CLEAR_AT_RESTART", "0") == "1": + print("Clearing database and file system at restart...") + db.drop_all() # Clear existing tables if configured + rmtree(Path("./results"), ignore_errors=True) # Clear results folder db.create_all() print("Database structure created successfully.") - print("Filling IHDR lookup table...") - create_crc_db() - print("IHDR table filled successfully.") + if getenv("SKIP_IHDR_FILL", "0") == "1": + print("Skipping IHDR lookup table fill as per configuration.") + else: + print("Filling IHDR lookup table...") + create_crc_db() + print("IHDR table filled successfully.") diff --git a/compose.dev.yml b/compose.dev.yml index c10943f..523ee2f 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -22,6 +22,7 @@ services: environment: - FLASK_DEBUG=true - FLASK_ENV=development + - SKIP_IHDR_FILL=1 worker: #image: ghcr.io/zeecka/aperisolve:latest From 531bdc9d816ba076f964b8738eef165017e9f3df Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 16:26:10 +0100 Subject: [PATCH 10/14] Sentry error handling if DSN fail --- aperisolve/sentry.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/aperisolve/sentry.py b/aperisolve/sentry.py index e0677a4..22042f1 100644 --- a/aperisolve/sentry.py +++ b/aperisolve/sentry.py @@ -25,18 +25,23 @@ def initialize_sentry() -> None: if not SENTRY_DSN or sentry_sdk.Hub.current.client is not None: return - sentry_sdk.init( - dsn=SENTRY_DSN, - integrations=[ - FlaskIntegration(), - SqlalchemyIntegration(), - RqIntegration(), - ThreadingIntegration(propagate_hub=True), - ], - traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE, - profiles_sample_rate=SENTRY_PROFILES_SAMPLE_RATE, - environment=SENTRY_ENVIRONMENT, - release=SENTRY_RELEASE, - send_default_pii=False, - enable_tracing=True, - ) + try: + sentry_sdk.init( + dsn=SENTRY_DSN, + integrations=[ + FlaskIntegration(), + SqlalchemyIntegration(), + RqIntegration(), + ThreadingIntegration(propagate_hub=True), + ], + traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE, + profiles_sample_rate=SENTRY_PROFILES_SAMPLE_RATE, + environment=SENTRY_ENVIRONMENT, + release=SENTRY_RELEASE, + send_default_pii=False, + enable_tracing=True, + ) + except sentry_sdk.utils.BadDsn as e: + print(f"Sentry DSN is invalid, skipping Sentry initialization: {e}") + except Exception as e: + print(f"Sentry initialization failed: {e}") From 7e1290c5d92fb33af2dc34bd53cdab5371d988c4 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 16:30:12 +0100 Subject: [PATCH 11/14] work still in progress --- aperisolve/models.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/aperisolve/models.py b/aperisolve/models.py index 6b300f8..9d82e8c 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -130,12 +130,13 @@ def init_db(app: Flask) -> None: - Prints status messages to console during initialization """ with app.app_context(): - if not db.engine.dialect.has_table(db.engine.connect(), "image"): - print(f"CLEAR_AT_RESTART={getenv('CLEAR_AT_RESTART', '0')}") - if getenv("CLEAR_AT_RESTART", "0") == "1": - print("Clearing database and file system at restart...") - db.drop_all() # Clear existing tables if configured - rmtree(Path("./results"), ignore_errors=True) # Clear results folder + if getenv("CLEAR_AT_RESTART", "0") == "1": # Force clear if CLEAR_AT_RESTART + print("Clearing database and file system at restart...") + db.drop_all() # Clear existing tables if configured + db.session.commit() # pylint: disable=no-member + rmtree(Path("./results"), ignore_errors=True) # Clear results folder + + if not db.engine.dialect.has_table(db.engine.connect(), "image"): # If not configured db.create_all() print("Database structure created successfully.") if getenv("SKIP_IHDR_FILL", "0") == "1": From ad9501a3b2dd2f56d9efc123766edaed17d71c41 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 20:49:24 +0100 Subject: [PATCH 12/14] Lint black --- aperisolve/models.py | 4 +++- aperisolve/sentry.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/aperisolve/models.py b/aperisolve/models.py index 9d82e8c..c38ad96 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -136,7 +136,9 @@ def init_db(app: Flask) -> None: db.session.commit() # pylint: disable=no-member rmtree(Path("./results"), ignore_errors=True) # Clear results folder - if not db.engine.dialect.has_table(db.engine.connect(), "image"): # If not configured + if not db.engine.dialect.has_table( + db.engine.connect(), "image" + ): # If not configured db.create_all() print("Database structure created successfully.") if getenv("SKIP_IHDR_FILL", "0") == "1": diff --git a/aperisolve/sentry.py b/aperisolve/sentry.py index 22042f1..13b5871 100644 --- a/aperisolve/sentry.py +++ b/aperisolve/sentry.py @@ -14,9 +14,12 @@ from sentry_sdk.integrations.threading import ThreadingIntegration SENTRY_DSN = os.environ.get("SENTRY_DSN") SENTRY_ENVIRONMENT = os.environ.get("SENTRY_ENVIRONMENT", "development") SENTRY_TRACES_SAMPLE_RATE = float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")) -SENTRY_PROFILES_SAMPLE_RATE = float(os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1")) +SENTRY_PROFILES_SAMPLE_RATE = float( + os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1") +) SENTRY_RELEASE = os.environ.get("SENTRY_RELEASE", "1.0.0") + def initialize_sentry() -> None: """Initialize Sentry SDK once, safely.""" From 60bed74a0de06275c2720f38f53438d2b46a7530 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 22:47:20 +0100 Subject: [PATCH 13/14] Now working --- aperisolve/app.py | 5 +++-- aperisolve/models.py | 26 ++++++++++++-------------- compose.dev.yml | 1 + compose.yml | 2 ++ 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index 295faaa..734097c 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -33,9 +33,10 @@ app.config["MAX_CONTENT_LENGTH"] = int( ) RESULT_FOLDER.mkdir(parents=True, exist_ok=True) - db.init_app(app) -init_db(app) +with app.app_context(): + init_db(app) + redis_conn = Redis(host="redis", port=6379) queue = Queue(connection=redis_conn) diff --git a/aperisolve/models.py b/aperisolve/models.py index c38ad96..dc47053 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -62,7 +62,7 @@ class IHDR(db.Model): # type: ignore packed = Column(BigInteger, nullable=False) -def create_crc_db() -> None: +def fill_ihdr_db() -> None: """ Creates a database of common PNG IHDR (Image Header) entries by generating combinations of standard PNG parameters such as resolutions, bit depths, @@ -132,18 +132,16 @@ def init_db(app: Flask) -> None: with app.app_context(): if getenv("CLEAR_AT_RESTART", "0") == "1": # Force clear if CLEAR_AT_RESTART print("Clearing database and file system at restart...") - db.drop_all() # Clear existing tables if configured - db.session.commit() # pylint: disable=no-member + db.session.remove() # pylint: disable=no-member + db.drop_all() rmtree(Path("./results"), ignore_errors=True) # Clear results folder - if not db.engine.dialect.has_table( - db.engine.connect(), "image" - ): # If not configured - db.create_all() - print("Database structure created successfully.") - if getenv("SKIP_IHDR_FILL", "0") == "1": - print("Skipping IHDR lookup table fill as per configuration.") - else: - print("Filling IHDR lookup table...") - create_crc_db() - print("IHDR table filled successfully.") + db.create_all() + print("Database structure created successfully.") + + if getenv("SKIP_IHDR_FILL", "0") == "1": + print("Skipping IHDR lookup table fill as per configuration.") + else: + print("Filling IHDR lookup table...") + fill_ihdr_db() + print("IHDR table filled successfully.") diff --git a/compose.dev.yml b/compose.dev.yml index 523ee2f..1bdd982 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -41,6 +41,7 @@ services: environment: - FLASK_DEBUG=true - FLASK_ENV=development + - CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing redis: image: redis:8 diff --git a/compose.yml b/compose.yml index 2afcb73..4489e66 100644 --- a/compose.yml +++ b/compose.yml @@ -37,6 +37,8 @@ services: - postgres env_file: - .env + environment: + - CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing postgres: image: postgres:18 From e525fd0774e3a242c7cb1bfb37aad0b5f5c79f52 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 22:55:58 +0100 Subject: [PATCH 14/14] Set mounted volume back --- README.md | 5 ++++- compose.dev.yml | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 48fa04c..285ca5f 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,10 @@ docker cp aperisolve-web:/aperisolve/results/filename.ext /path/to/backup/filena ``` > [!WARNING] -> Uploaded files are stored in a tmpfs. This means that they are stored in memory and will be lost if the server is restarted. Ensure that you back up any important files before the server is stopped or restarted. +> If switching between dev and production compose files, remove the `results` directory or mounted volume to avoid conflicts: +> ```bash +> rm -rf aperisolve/results +> ``` ### Adding a new analyzer diff --git a/compose.dev.yml b/compose.dev.yml index 1bdd982..134bea6 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -9,7 +9,6 @@ services: - "5000:5000" volumes: - ./aperisolve:/aperisolve - - apresults:/aperisolve/results depends_on: postgres: condition: service_healthy @@ -32,7 +31,6 @@ services: command: ["/bin/sh", "-c", "rq worker default --url redis://redis:6379/0"] volumes: - ./aperisolve:/aperisolve - - apresults:/aperisolve/results # Shared named volume (renamed) depends_on: - redis - postgres