diff --git a/.env b/.env index 4116ee9..366a17d 100644 --- a/.env +++ b/.env @@ -1,15 +1,26 @@ +# Application configuration MAX_CONTENT_LENGTH=1048576 MAX_PENDING_TIME=600 MAX_STORE_TIME=259200 +CLEAR_AT_RESTART=1 +SKIP_IHDR_FILL=1 + +# Flask configuration FLASK_DEBUG=0 FLASK_ENV=production -DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve + +# Redis configuration RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0 + +# Database configuration POSTGRES_DB=aperisolve POSTGRES_USER=aperiuser POSTGRES_PASSWORD=aperipass +DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve + +# Sentry configuration SENTRY_DSN=https://your-dsn@sentry.io/project-id SENTRY_TRACES_SAMPLE_RATE=0.1 SENTRY_PROFILES_SAMPLE_RATE=0.1 SENTRY_RELEASE=1.0.0 -ENVIRONMENT=production \ No newline at end of file +SENTRY_ENVIRONMENT=production diff --git a/README.md b/README.md index 929dbc2..285ca5f 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Support Aperi'Solve: Aperi'Solve is an open-source steganalysis web platform that performs automated analysis on images to detect and extract hidden data using common steganography tools and techniques. Table of contents + - [Key features](#key-features) - [Quick start (Docker)](#quick-start-docker) - [Development](#development) @@ -59,6 +60,7 @@ In case you want to host your own version of https://www.aperisolve.com/. Recommended: Docker + Docker Compose. Clone and start the full stack (production-like): + ```bash git clone https://github.com/Zeecka/AperiSolve.git cd AperiSolve @@ -68,13 +70,16 @@ docker compose up -d Default: http://localhost:5000/ ## Development + Development compose: + ```bash # development environment (hot reload and local volumes) docker compose -f compose.dev.yml up --build ``` Useful commands: + ```bash # Stop and remove containers, networks and volumes (results are stored in a volume) docker compose down -v @@ -84,6 +89,12 @@ docker exec -it aperisolve-web bash # Enter Postgres shell (from host) docker exec -it postgres psql -U aperiuser -d aperisolve + +# Backup all uploaded files +docker cp -r aperisolve-web:/aperisolve/results /path/to/backup/location + +# Backup a single uploaded file +docker cp aperisolve-web:/aperisolve/results/filename.ext /path/to/backup/filename.ext ``` > [!WARNING] @@ -97,8 +108,10 @@ docker exec -it postgres psql -U aperiuser -d aperisolve Adding a custom analyzer is straightforward: 1. Create your analyzer file: + - Copy `aperisolve/analyzers/template_analyzer.py` -> `aperisolve/analyzers/myanalyzer.py` - Implement function signature similar to: + ```python # aperisolve/analyzers/myanalyzer.py def analyze_myanalyzer(image_path: str, results_dir: str) -> dict: @@ -111,7 +124,9 @@ Adding a custom analyzer is straightforward: ``` 2. Register the analyzer in the worker pipeline: + - Edit `aperisolve/workers.py`: + ```python # import from .analyzers.myanalyzer import analyze_myanalyzer @@ -126,23 +141,27 @@ Adding a custom analyzer is straightforward: ``` 3. Add your analyzer to the UI order: + - Edit `aperisolve/static/js/aperisolve.js` and append `myanalyzer` to `TOOL_ORDER` so it appears in the frontend. 4. Test locally: run the worker and submit jobs to ensure outputs are produced and displayed. > [!TIP] +> > - Keep analyzers idempotent and write outputs to the provided `results_dir`. > - Return structured JSON so the frontend can render links/downloads automatically. ## Configuration & environment variables Typical services: + - Web app (Flask) - Workers (Python) - Redis (RQ) - PostgreSQL Main environment variables (examples): + - DATABASE_URL=postgresql://aperiuser:password@postgres:5432/aperisolve - REDIS_URL=redis://redis:6379/0 - SECRET_KEY=change_me @@ -171,11 +190,7 @@ This separation keeps heavy tools (binwalk, foremost, zsteg, etc.) isolated and ## Roadmap -- [ ] **[Bug]** Error 500 when file already exist on disc - [ ] **[Bug]** Duplicate key value violates unique constraint "submission_pkey" -- [ ] **[Bug]** Binwalk Extractor error -- [ ] **[Feature]** Implement healthcheck and auto reboot -- [ ] **[Feature]** Implement Sentry for better error handling - [ ] **[Feature]** Zsteg: full extraction (--all) and download of discovered files (mp3, etc.) - [ ] **[Feature]** Mobile-friendly UI / test on mobile - [ ] **[Feature]** i18n (internationalization) @@ -185,12 +200,14 @@ This separation keeps heavy tools (binwalk, foremost, zsteg, etc.) isolated and ## Contributing Contributions are welcome! Please: + 1. Fork the repository 2. Create a feature branch 3. Open a pull request describing your change > [!IMPORTANT] > Please follow the code style and run linters before submitting. The project adheres to: +> > - Black > - Flake8 (ignoring E203, E501, W503) > - Pylint (ignoring W0718, R0903, R0801) @@ -206,4 +223,5 @@ CI will run these checks on each PR. ## Credits Acknowledgements: + - Thanks to contributors and the open-source community for the tools integrated ([zsteg](https://github.com/zed-0xff/zsteg), [steghide](https://steghide.sourceforge.net/), [outguess](https://www.rbcafe.com/software/outguess/), [openstego](https://www.openstego.com/), [exiftool](https://exiftool.org/), [binwalk](https://github.com/ReFirmLabs/binwalk), [foremost](https://foremost.sourceforge.net/), [pngcheck](https://www.libpng.org/pub/png/apps/pngcheck.html), [strings](https://pubs.opengroup.org/onlinepubs/9799919799/utilities/strings.html), ...). diff --git a/aperisolve/app.py b/aperisolve/app.py index 295faaa..734097c 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -33,9 +33,10 @@ app.config["MAX_CONTENT_LENGTH"] = int( ) RESULT_FOLDER.mkdir(parents=True, exist_ok=True) - db.init_app(app) -init_db(app) +with app.app_context(): + init_db(app) + redis_conn = Redis(host="redis", port=6379) queue = Queue(connection=redis_conn) diff --git a/aperisolve/models.py b/aperisolve/models.py index b064689..dc47053 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -5,6 +5,9 @@ import itertools from datetime import datetime, timezone +from os import getenv +from pathlib import Path +from shutil import rmtree from flask import Flask from flask_sqlalchemy import SQLAlchemy @@ -59,7 +62,7 @@ class IHDR(db.Model): # type: ignore packed = Column(BigInteger, nullable=False) -def create_crc_db() -> None: +def fill_ihdr_db() -> None: """ Creates a database of common PNG IHDR (Image Header) entries by generating combinations of standard PNG parameters such as resolutions, bit depths, @@ -127,10 +130,18 @@ def init_db(app: Flask) -> None: - Prints status messages to console during initialization """ with app.app_context(): - if not db.engine.dialect.has_table(db.engine.connect(), "image"): - print("Creating database...") - db.create_all() - print("Database structure created successfully.") + if getenv("CLEAR_AT_RESTART", "0") == "1": # Force clear if CLEAR_AT_RESTART + print("Clearing database and file system at restart...") + db.session.remove() # pylint: disable=no-member + db.drop_all() + rmtree(Path("./results"), ignore_errors=True) # Clear results folder + + db.create_all() + print("Database structure created successfully.") + + if getenv("SKIP_IHDR_FILL", "0") == "1": + print("Skipping IHDR lookup table fill as per configuration.") + else: print("Filling IHDR lookup table...") - create_crc_db() + fill_ihdr_db() print("IHDR table filled successfully.") diff --git a/aperisolve/sentry.py b/aperisolve/sentry.py index bb87683..13b5871 100644 --- a/aperisolve/sentry.py +++ b/aperisolve/sentry.py @@ -12,35 +12,39 @@ from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration from sentry_sdk.integrations.threading import ThreadingIntegration SENTRY_DSN = os.environ.get("SENTRY_DSN") -ENVIRONMENT = os.environ.get("ENVIRONMENT", "development") +SENTRY_ENVIRONMENT = os.environ.get("SENTRY_ENVIRONMENT", "development") +SENTRY_TRACES_SAMPLE_RATE = float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")) +SENTRY_PROFILES_SAMPLE_RATE = float( + os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1") +) +SENTRY_RELEASE = os.environ.get("SENTRY_RELEASE", "1.0.0") def initialize_sentry() -> None: """Initialize Sentry SDK once, safely.""" # No DSN → do nothing - dsn = os.environ.get("SENTRY_DSN") - if not dsn: - return - # Prevent double initialization (Flask + RQ, CLI, tests, etc.) - if sentry_sdk.Hub.current.client is not None: + if not SENTRY_DSN or sentry_sdk.Hub.current.client is not None: return - sentry_sdk.init( - dsn=dsn, - integrations=[ - FlaskIntegration(), - SqlalchemyIntegration(), - RqIntegration(), - ThreadingIntegration(propagate_hub=True), - ], - traces_sample_rate=float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")), - profiles_sample_rate=float( - os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1") - ), - environment=os.environ.get("ENVIRONMENT", "development"), - release=os.environ.get("SENTRY_RELEASE", "1.0.0"), - send_default_pii=False, - enable_tracing=True, - ) + try: + sentry_sdk.init( + dsn=SENTRY_DSN, + integrations=[ + FlaskIntegration(), + SqlalchemyIntegration(), + RqIntegration(), + ThreadingIntegration(propagate_hub=True), + ], + traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE, + profiles_sample_rate=SENTRY_PROFILES_SAMPLE_RATE, + environment=SENTRY_ENVIRONMENT, + release=SENTRY_RELEASE, + send_default_pii=False, + enable_tracing=True, + ) + except sentry_sdk.utils.BadDsn as e: + print(f"Sentry DSN is invalid, skipping Sentry initialization: {e}") + except Exception as e: + print(f"Sentry initialization failed: {e}") diff --git a/aperisolve/static/js/aperisolve.js b/aperisolve/static/js/aperisolve.js index 0090c17..0294ead 100644 --- a/aperisolve/static/js/aperisolve.js +++ b/aperisolve/static/js/aperisolve.js @@ -486,7 +486,9 @@ if (browseBtn) { } catch (e) { showDanger("❌ Invalid server response.", true); } - } else { + } else if (xhr.status == 413) { + showDanger("❌ File too large. Please upload a smaller file.", true); + }else { showDanger(`❌ HTTP error ${xhr.status}`, true); } progressBar.style.width = "100%"; diff --git a/compose.dev.yml b/compose.dev.yml index e23163d..134bea6 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -4,7 +4,7 @@ services: build: . container_name: aperisolve-web restart: always - command: ["flask", "--app", "aperisolve.app", "run", "--host=0.0.0.0", "--port=5000"] + command: ["/bin/sh", "-c", "flask --app aperisolve.app run --host=0.0.0.0 --port=5000"] ports: - "5000:5000" volumes: @@ -21,13 +21,14 @@ services: environment: - FLASK_DEBUG=true - FLASK_ENV=development + - SKIP_IHDR_FILL=1 worker: #image: ghcr.io/zeecka/aperisolve:latest build: . container_name: aperisolve-worker restart: always - command: rq worker default --url redis://redis:6379/0 + command: ["/bin/sh", "-c", "rq worker default --url redis://redis:6379/0"] volumes: - ./aperisolve:/aperisolve depends_on: @@ -38,20 +39,21 @@ services: environment: - FLASK_DEBUG=true - FLASK_ENV=development + - CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing redis: - image: redis:7 + image: redis:8 container_name: redis restart: always env_file: - .env postgres: - image: postgres:14 + image: postgres:18 container_name: postgres restart: always volumes: - - pgdata:/var/lib/postgresql/data + - pgdata:/var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s @@ -60,7 +62,6 @@ services: env_file: - .env - rqdashboard: image: eoranged/rq-dashboard container_name: rq-dashboard @@ -71,6 +72,7 @@ services: - redis env_file: - .env - + volumes: - pgdata: \ No newline at end of file + apresults: + pgdata: diff --git a/compose.yml b/compose.yml index 777fef2..4489e66 100644 --- a/compose.yml +++ b/compose.yml @@ -17,6 +17,12 @@ services: condition: service_started env_file: - .env + healthcheck: + test: ["CMD-SHELL", "dd if=/dev/urandom of=/tmp/healthcheck.png bs=1 count=1000000 && curl -X POST -F 'image=@/tmp/healthcheck.png' http://localhost:5000/upload -s -o /dev/null -w '%{http_code}' | grep -q 200"] + interval: 3600s + timeout: 30s + retries: 3 + start_period: 900s worker: image: ghcr.io/zeecka/aperisolve:latest @@ -31,13 +37,15 @@ services: - postgres env_file: - .env + environment: + - CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing postgres: - image: postgres:14 + image: postgres:18 container_name: postgres restart: always volumes: - - pgdata:/var/lib/postgresql/data + - pgdata:/var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s @@ -63,7 +71,7 @@ services: - redis env_file: - .env - + volumes: apresults: pgdata: