From d0c2f2480044a974b5ec16fcceff0f3a6dd6a83e Mon Sep 17 00:00:00 2001 From: Zeecka Date: Sun, 28 Dec 2025 10:33:46 +0100 Subject: [PATCH 001/102] WIP --- Dockerfile | 35 +++++++-------- aperisolve/analyzers/binwalk.py | 8 ++-- aperisolve/static/js/aperisolve.js | 72 +++++++++++++++--------------- 3 files changed, 56 insertions(+), 59 deletions(-) diff --git a/Dockerfile b/Dockerfile index fe69aa9..1ac3310 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,22 +3,14 @@ FROM python:3.14-slim AS build WORKDIR / # Install build tools and dependencies -RUN apt-get update && apt-get install -y \ - wget \ - default-jre \ - ruby \ - zip \ - 7zip -# Install steganography and forensics tools -RUN apt-get update && apt-get install -y \ - binwalk \ - foremost \ - exiftool \ - steghide \ - binutils \ - outguess \ - pngcheck \ + +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget default-jre ruby \ + bzip2 xz-utils gzip lzip lzma lzop tar unzip p7zip-full squashfs-tools \ + file qpdf cpio arj cabextract sharutils lz4 ccache mtd-utils \ + python3-lxml upx-ucl zlib1g-dev liblzma-dev \ + binwalk foremost exiftool steghide binutils outguess pngcheck \ && gem install zsteg # Install OpenStego @@ -26,17 +18,20 @@ RUN wget https://github.com/syvaidya/openstego/releases/download/openstego-0.8.6 dpkg -i /tmp/openstego.deb && \ rm /tmp/openstego.deb -# Clean up apt cache and remove useless packages -RUN apt-get remove -y wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - +# Copy application COPY aperisolve/ /aperisolve/ RUN mkdir -p /data +# Install Python dependencies RUN pip install --no-cache-dir -r /aperisolve/requirements.txt ENV PYTHONUNBUFFERED=1 +# Auto clean after build +RUN apt-get remove -y wget && \ + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "--access-logfile", "-", "--error-logfile", "-", "--log-level", "info", "--capture-output", "aperisolve.wsgi:application"] diff --git a/aperisolve/analyzers/binwalk.py b/aperisolve/analyzers/binwalk.py index 32d6fec..823590c 100644 --- a/aperisolve/analyzers/binwalk.py +++ b/aperisolve/analyzers/binwalk.py @@ -3,6 +3,7 @@ import shutil import subprocess from pathlib import Path +from typing import Any from .utils import MAX_PENDING_TIME, update_data @@ -17,7 +18,7 @@ def analyze_binwalk(input_img: Path, output_dir: Path) -> None: stderr = "" # Run binwalk data = subprocess.run( - ["binwalk", "-e", "../" + str(image_name), "--run-as=root"], + ["binwalk", "--matryoshka", "-e", "../" + str(image_name), "--run-as=root"], cwd=output_dir, capture_output=True, text=True, @@ -45,16 +46,17 @@ def analyze_binwalk(input_img: Path, output_dir: Path) -> None: shutil.rmtree(extracted_dir) if len(stderr) > 0: - err = { + err: dict[str, dict[str, Any]] = { "binwalk": { "status": "error", + "output": data.stdout.split("\n") if data else [], "error": stderr, } } update_data(output_dir, err) return None - output_data = { + output_data: dict[str, dict[str, Any]] = { "binwalk": { "status": "ok", "output": data.stdout.split("\n") if data else [], diff --git a/aperisolve/static/js/aperisolve.js b/aperisolve/static/js/aperisolve.js index a093ce7..d7c1e89 100644 --- a/aperisolve/static/js/aperisolve.js +++ b/aperisolve/static/js/aperisolve.js @@ -252,6 +252,38 @@ function parseResult(result) { analyzer.innerHTML += `

${capitalize(tool.replace("_"," "))}

`; + // Parse text output + if (typeof result[tool]["output"] === "string") { + analyzer.innerHTML += `
${escapeHtml(
+        result[tool]["output"]
+      )}
`; + } else if (Array.isArray(result[tool]["output"])) { + if (result[tool]["output"].length > 0) { + var texarea_content = `
`; + texarea_content += ``; + texarea_content += ``; + texarea_content += `
`; + analyzer.innerHTML += texarea_content; + } + } else if (typeof result[tool]["output"] === "object") { + var table_content = `
`; + table_content += ``; + for (const key in result[tool]["output"]) { + table_content += ``; + table_content += ``; + } + table_content += `
${escapeHtml(key)}${escapeHtml( + result[tool]["output"][key] + )}
`; + table_content += ``; + analyzer.innerHTML += table_content; + } + + // Parse images, downloads, ... if (result[tool]["status"] === "ok") { if ("images" in result[tool]) { // Parse image output @@ -286,48 +318,16 @@ function parseResult(result) { } } - // Parse text output - - - if (typeof result[tool]["output"] === "string") { - analyzer.innerHTML += `
${escapeHtml(
-          result[tool]["output"]
-        )}
`; - } else if (Array.isArray(result[tool]["output"])) { - if (result[tool]["output"].length > 0) { - var texarea_content = `
`; - texarea_content += ``; - texarea_content += ``; - texarea_content += `
`; - analyzer.innerHTML += texarea_content; - } - } else if (typeof result[tool]["output"] === "object") { - var table_content = `
`; - table_content += ``; - for (const key in result[tool]["output"]) { - table_content += ``; - table_content += ``; - } - table_content += `
${escapeHtml(key)}${escapeHtml( - result[tool]["output"][key] - )}
`; - table_content += ``; - analyzer.innerHTML += table_content; - } else { - // analyzer.innerHTML += `
${result[tool]["output"]}
`; - } - if ("download" in result[tool]) { // Parse download link analyzer.innerHTML += `
Download file`; } - } else if (result[tool]["status"] === "error") { + } + + // Parse errors + if (result[tool]["status"] === "error") { showDanger(result[tool]["error"]); } } From c0e95bd2ced3293b16a44a0750cc546d9eaccf20 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Sun, 28 Dec 2025 18:57:33 +0100 Subject: [PATCH 002/102] Update readme roadmap --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 467109a..929dbc2 100644 --- a/README.md +++ b/README.md @@ -172,9 +172,8 @@ This separation keeps heavy tools (binwalk, foremost, zsteg, etc.) isolated and ## Roadmap - [ ] **[Bug]** Error 500 when file already exist on disc -- [ ] **[Bug]** Multiple bug on buggy Jpeg filebugSomething isn't working -- [ ] **[Bug]** Binwalk Extractor errorbugSomething isn't working -- [ ] **[Bug]** Grayscale on a RGB image ?bugSomething isn't working +- [ ] **[Bug]** Duplicate key value violates unique constraint "submission_pkey" +- [ ] **[Bug]** Binwalk Extractor error - [ ] **[Feature]** Implement healthcheck and auto reboot - [ ] **[Feature]** Implement Sentry for better error handling - [ ] **[Feature]** Zsteg: full extraction (--all) and download of discovered files (mp3, etc.) From 566055e5eb39314a9adeedd752bfe762978070d7 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Tue, 30 Dec 2025 06:49:05 +0530 Subject: [PATCH 003/102] Fix Dockerfile to install default-jdk instead of default-jre to get jar --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fe69aa9..41e4c21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ WORKDIR / # Install build tools and dependencies RUN apt-get update && apt-get install -y \ wget \ - default-jre \ + default-jdk \ ruby \ zip \ 7zip From 54adcbcbfa61faf02dceb9e72db37153ebcdb267 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Tue, 30 Dec 2025 06:49:34 +0530 Subject: [PATCH 004/102] Fix error handling in binwalk analyzer to check for zip existence before reporting errors --- aperisolve/analyzers/binwalk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aperisolve/analyzers/binwalk.py b/aperisolve/analyzers/binwalk.py index 32d6fec..c3d1f2e 100644 --- a/aperisolve/analyzers/binwalk.py +++ b/aperisolve/analyzers/binwalk.py @@ -44,7 +44,7 @@ def analyze_binwalk(input_img: Path, output_dir: Path) -> None: if extracted_dir.exists(): shutil.rmtree(extracted_dir) - if len(stderr) > 0: + if len(stderr) > 0 and not zip_exist: err = { "binwalk": { "status": "error", From a2f6579380c310e89dcd032001947029c7884b3e Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Tue, 30 Dec 2025 06:50:41 +0530 Subject: [PATCH 005/102] Clarify error reporting logic in binwalk analyzer comments --- aperisolve/analyzers/binwalk.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aperisolve/analyzers/binwalk.py b/aperisolve/analyzers/binwalk.py index c3d1f2e..69920e7 100644 --- a/aperisolve/analyzers/binwalk.py +++ b/aperisolve/analyzers/binwalk.py @@ -44,6 +44,8 @@ def analyze_binwalk(input_img: Path, output_dir: Path) -> None: if extracted_dir.exists(): shutil.rmtree(extracted_dir) + # Only report error if stderr exists AND extraction failed. + # If zip_exist is True, binwalk worked (despite warnings), so we proceed. if len(stderr) > 0 and not zip_exist: err = { "binwalk": { From ea04b57dcea011a7bbfc8ecd8d631b66676f4ff0 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Tue, 30 Dec 2025 11:48:05 +0530 Subject: [PATCH 006/102] Refactor upload_image function to streamline submission and image handling logic --- aperisolve/app.py | 116 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 86 insertions(+), 30 deletions(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index e81de53..9705156 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -112,21 +112,26 @@ def upload_image() -> tuple[Response, int]: submission_hash = hashlib.md5(submission_data).hexdigest() submission_path = RESULT_FOLDER / img_hash / submission_hash - # If submission already exists, return submission hash - if submission_path.exists(): - submission: Submission = Submission.query.filter_by( - hash=submission_hash - ).first() - return jsonify({"submission_hash": submission.hash}), 200 + submission: Submission = Submission.query.filter_by( + hash=submission_hash + ).first() - # If image is new, create a new Image entry + if submission_path.exists() and submission is not None: + return jsonify({"submission_hash": submission.hash}),200 + new_img_path = RESULT_FOLDER / img_hash / image_name - if not new_img_path.parent.exists(): - new_img_path.parent.mkdir(parents=True, exist_ok=True) - with open(new_img_path, "wb") as f: # Write file to disk - f.write(image_data) - new_img = Image( + sub_img = Image.query.filter_by( + hash=img_hash + ).first() + + if sub_img is None: + if not new_img_path.parent.exists() or not new_img_path.exists(): + new_img_path.parent.mkdir(parents=True, exist_ok=True) + with open(new_img_path,"wb") as f: + f.write(image_data) + + sub_img = Image( file=str(new_img_path), hash=img_hash, size=len(image_data), @@ -134,31 +139,82 @@ def upload_image() -> tuple[Response, int]: first_submission_date=datetime.now(timezone.utc), last_submission_date=datetime.now(timezone.utc), ) - db.session.add(new_img) # pylint: disable=no-member - db.session.commit() # pylint: disable=no-member + db.session.add(sub_img) + db.session.commit() - sub_img = Image.query.filter_by(hash=img_hash).first() # type: ignore + sub_img.upload_count += 1 - db.session.commit() # pylint: disable=no-member + db.session.commit() - # Create new Submission entry submission_path.mkdir(parents=True, exist_ok=True) - submission = Submission( - filename=image.filename, - password=password, - deep_analysis=deep_analysis, - hash=submission_hash, - status="pending", - date=time.time(), - image_hash=sub_img.hash, # type: ignore - ) - db.session.add(submission) # pylint: disable=no-member - db.session.commit() # pylint: disable=no-member - # Start the analysis jobs + if not submission: + submission = Submission( + filename=image.filename, + password=password, + deep_analysis=deep_analysis, + hash=submission_hash, + status="pending", + date=time.time(), + image_hash=sub_img.hash, + ) + db.session.add(submission) + db.session.commit() + queue.enqueue("aperisolve.workers.analyze_image", submission.hash, job_timeout=300) - return jsonify({"submission_hash": submission.hash}), 200 + return jsonify({"submission_hash": submission.hash}), 20 + + + + + # # If submission already exists, return submission hash + # if submission_path.exists(): + # submission: Submission = Submission.query.filter_by( + # hash=submission_hash + # ).first() + # return jsonify({"submission_hash": submission.hash}), 200 + + # # If image is new, create a new Image entry + # new_img_path = RESULT_FOLDER / img_hash / image_name + # if not new_img_path.parent.exists(): + # new_img_path.parent.mkdir(parents=True, exist_ok=True) + # with open(new_img_path, "wb") as f: # Write file to disk + # f.write(image_data) + + # new_img = Image( + # file=str(new_img_path), + # hash=img_hash, + # size=len(image_data), + # upload_count=0, + # first_submission_date=datetime.now(timezone.utc), + # last_submission_date=datetime.now(timezone.utc), + # ) + # db.session.add(new_img) # pylint: disable=no-member + # db.session.commit() # pylint: disable=no-member + + # sub_img = Image.query.filter_by(hash=img_hash).first() # type: ignore + # sub_img.upload_count += 1 + # db.session.commit() # pylint: disable=no-member + + # # Create new Submission entry + # submission_path.mkdir(parents=True, exist_ok=True) + # submission = Submission( + # filename=image.filename, + # password=password, + # deep_analysis=deep_analysis, + # hash=submission_hash, + # status="pending", + # date=time.time(), + # image_hash=sub_img.hash, # type: ignore + # ) + # db.session.add(submission) # pylint: disable=no-member + + # db.session.commit() # pylint: disable=no-member + # # Start the analysis jobs + # queue.enqueue("aperisolve.workers.analyze_image", submission.hash, job_timeout=300) + + # return jsonify({"submission_hash": submission.hash}), 200 @app.route("/status/", methods=["GET"]) From 8e72911f54beec57b3abf04a9641e3b220808b38 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Tue, 30 Dec 2025 11:48:20 +0530 Subject: [PATCH 007/102] Fix upload_image response status code from 20 to 200 --- aperisolve/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index 9705156..94b3f54 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -163,7 +163,7 @@ def upload_image() -> tuple[Response, int]: queue.enqueue("aperisolve.workers.analyze_image", submission.hash, job_timeout=300) - return jsonify({"submission_hash": submission.hash}), 20 + return jsonify({"submission_hash": submission.hash}), 200 From 4a380d277e883163ae304f83d152810af2de76c6 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Tue, 30 Dec 2025 11:59:02 +0530 Subject: [PATCH 008/102] simple documentation comments --- aperisolve/app.py | 65 ++++++++++------------------------------------- 1 file changed, 14 insertions(+), 51 deletions(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index 94b3f54..0ebc831 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -112,25 +112,33 @@ def upload_image() -> tuple[Response, int]: submission_hash = hashlib.md5(submission_data).hexdigest() submission_path = RESULT_FOLDER / img_hash / submission_hash + # Check if hash is present on the DB submission: Submission = Submission.query.filter_by( hash=submission_hash ).first() + # Only return if BOTH the file exists AND the DB entry exists if submission_path.exists() and submission is not None: return jsonify({"submission_hash": submission.hash}),200 + + # Self-Healing Logic new_img_path = RESULT_FOLDER / img_hash / image_name + # Try to find image in DB by hash sub_img = Image.query.filter_by( hash=img_hash ).first() + # If DB entry is missing (even if file exists on disk), create it! if sub_img is None: if not new_img_path.parent.exists() or not new_img_path.exists(): new_img_path.parent.mkdir(parents=True, exist_ok=True) with open(new_img_path,"wb") as f: f.write(image_data) + + # Recreate the missing Database Entry sub_img = Image( file=str(new_img_path), hash=img_hash, @@ -142,12 +150,15 @@ def upload_image() -> tuple[Response, int]: db.session.add(sub_img) db.session.commit() - + # Recreate the missing Database Entry sub_img.upload_count += 1 db.session.commit() + # Create new Submission entry submission_path.mkdir(parents=True, exist_ok=True) + # Only create a new submission if it does not already exist + # New submission for new image if not submission: submission = Submission( filename=image.filename, @@ -161,62 +172,14 @@ def upload_image() -> tuple[Response, int]: db.session.add(submission) db.session.commit() + # Start the analysis jobs queue.enqueue("aperisolve.workers.analyze_image", submission.hash, job_timeout=300) + # Return submission identifier return jsonify({"submission_hash": submission.hash}), 200 - - # # If submission already exists, return submission hash - # if submission_path.exists(): - # submission: Submission = Submission.query.filter_by( - # hash=submission_hash - # ).first() - # return jsonify({"submission_hash": submission.hash}), 200 - - # # If image is new, create a new Image entry - # new_img_path = RESULT_FOLDER / img_hash / image_name - # if not new_img_path.parent.exists(): - # new_img_path.parent.mkdir(parents=True, exist_ok=True) - # with open(new_img_path, "wb") as f: # Write file to disk - # f.write(image_data) - - # new_img = Image( - # file=str(new_img_path), - # hash=img_hash, - # size=len(image_data), - # upload_count=0, - # first_submission_date=datetime.now(timezone.utc), - # last_submission_date=datetime.now(timezone.utc), - # ) - # db.session.add(new_img) # pylint: disable=no-member - # db.session.commit() # pylint: disable=no-member - - # sub_img = Image.query.filter_by(hash=img_hash).first() # type: ignore - # sub_img.upload_count += 1 - # db.session.commit() # pylint: disable=no-member - - # # Create new Submission entry - # submission_path.mkdir(parents=True, exist_ok=True) - # submission = Submission( - # filename=image.filename, - # password=password, - # deep_analysis=deep_analysis, - # hash=submission_hash, - # status="pending", - # date=time.time(), - # image_hash=sub_img.hash, # type: ignore - # ) - # db.session.add(submission) # pylint: disable=no-member - - # db.session.commit() # pylint: disable=no-member - # # Start the analysis jobs - # queue.enqueue("aperisolve.workers.analyze_image", submission.hash, job_timeout=300) - - # return jsonify({"submission_hash": submission.hash}), 200 - - @app.route("/status/", methods=["GET"]) def get_status(hash_val: str) -> Response: """Get the processing status of a submission.""" From a0c616bc78622d4e4ee572d307e90af0a97994d5 Mon Sep 17 00:00:00 2001 From: Aradhya C P <135510032+aradhyacp@users.noreply.github.com> Date: Tue, 30 Dec 2025 13:00:05 +0530 Subject: [PATCH 009/102] Update aperisolve/app.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- aperisolve/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index 0ebc831..55f321f 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -150,7 +150,7 @@ def upload_image() -> tuple[Response, int]: db.session.add(sub_img) db.session.commit() - # Recreate the missing Database Entry + # Increment upload count for the image sub_img.upload_count += 1 db.session.commit() From 9f21ac429dceb403cf19314a2e66a37ca055d6d9 Mon Sep 17 00:00:00 2001 From: Aradhya C P <135510032+aradhyacp@users.noreply.github.com> Date: Tue, 30 Dec 2025 13:00:12 +0530 Subject: [PATCH 010/102] Update aperisolve/app.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- aperisolve/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index 55f321f..f820226 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -134,7 +134,7 @@ def upload_image() -> tuple[Response, int]: if sub_img is None: if not new_img_path.parent.exists() or not new_img_path.exists(): new_img_path.parent.mkdir(parents=True, exist_ok=True) - with open(new_img_path,"wb") as f: + with open(new_img_path, "wb") as f: f.write(image_data) From 87df3cc629be451e7afb2cc17f447ee4a99480f2 Mon Sep 17 00:00:00 2001 From: Aradhya C P <135510032+aradhyacp@users.noreply.github.com> Date: Tue, 30 Dec 2025 13:00:31 +0530 Subject: [PATCH 011/102] Update aperisolve/app.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- aperisolve/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index f820226..75359d2 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -119,7 +119,7 @@ def upload_image() -> tuple[Response, int]: # Only return if BOTH the file exists AND the DB entry exists if submission_path.exists() and submission is not None: - return jsonify({"submission_hash": submission.hash}),200 + return jsonify({"submission_hash": submission.hash}), 200 # Self-Healing Logic From ffd97c39551ec4033e4005e0ea35e517944af317 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Tue, 30 Dec 2025 17:34:44 +0530 Subject: [PATCH 012/102] Add Sentry configuration to .env file --- .env | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.env b/.env index 7d7e694..4116ee9 100644 --- a/.env +++ b/.env @@ -8,3 +8,8 @@ RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0 POSTGRES_DB=aperisolve POSTGRES_USER=aperiuser POSTGRES_PASSWORD=aperipass +SENTRY_DSN=https://your-dsn@sentry.io/project-id +SENTRY_TRACES_SAMPLE_RATE=0.1 +SENTRY_PROFILES_SAMPLE_RATE=0.1 +SENTRY_RELEASE=1.0.0 +ENVIRONMENT=production \ No newline at end of file From b7d25522f11a20d31eaf4d11d9d18fa625f383f7 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Tue, 30 Dec 2025 17:35:32 +0530 Subject: [PATCH 013/102] Integrate Sentry for error tracking and performance monitoring --- aperisolve/app.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/aperisolve/app.py b/aperisolve/app.py index 75359d2..5b0ad4d 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -8,6 +8,11 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Optional +import sentry_sdk +from sentry_sdk.integrations.flask import FlaskIntegration +from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration +from sentry_sdk.integrations.rq import RqIntegration + from flask import Flask, Response, abort, jsonify, render_template, request, send_file from redis import Redis from rq import Queue @@ -16,6 +21,32 @@ from .cleanup import cleanup_old_entries from .config import IMAGE_EXTENSIONS, RESULT_FOLDER, WORKER_FILES from .models import Image, Submission, db, init_db +SENTRY_DSN = os.environ.get("SENTRY_DSN") +ENVIRONMENT = os.environ.get("ENVIRONMENT", "development") + +if SENTRY_DSN: + sentry_sdk.init( + dsn=SENTRY_DSN, + integrations=[ + FlaskIntegration(), + SqlalchemyIntegration(), + RqIntegration(), + ], + # Set traces_sample_rate to 1.0 to capture 100% of transactions for performance monitoring. + # Adjust this value in production (e.g., 0.1 for 10%) + traces_sample_rate=float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")), + # Set profiles_sample_rate to 1.0 to profile 100% of sampled transactions. + profiles_sample_rate=float(os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1")), + # Environment tag (production, staging, development) + environment=ENVIRONMENT, + # Release version (use git commit hash or version number) + release=os.environ.get("SENTRY_RELEASE", "1.0.0"), + # Send default PII (user IPs, etc.) - set to False for privacy + send_default_pii=False, + # Enable automatic breadcrumb tracking + enable_tracing=True, + ) + app: Flask = Flask(__name__) app.json.sort_keys = False # type: ignore app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("DB_URI") @@ -35,6 +66,11 @@ queue = Queue(connection=redis_conn) @app.errorhandler(413) def too_large(_: Any) -> tuple[Response, int]: """Error Handler for Max File Size.""" + sentry_sdk.set_context("upload", { + "content_length": request.content_length, + "max_allowed": app.config["MAX_CONTENT_LENGTH"], + }) + sentry_sdk.capture_message("Upload failed: 413 Payload Too Large", level="warning") return jsonify({"error": "Image size exceeded"}), 413 @@ -43,6 +79,7 @@ def not_found(_: Any) -> str: """Error Handler for 404 not found.""" return render_template("error.html", message="Resource not found", statuscode=404) +# Sentry automatically captures 500 errors @app.route("/") def index() -> str: From 149d80fb473851010f0397f52d385f0f81edbdbe Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Tue, 30 Dec 2025 17:35:44 +0530 Subject: [PATCH 014/102] Enhance Sentry integration in workers for improved error tracking and performance monitoring --- aperisolve/workers.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/aperisolve/workers.py b/aperisolve/workers.py index c5be8c7..d577fa2 100644 --- a/aperisolve/workers.py +++ b/aperisolve/workers.py @@ -1,9 +1,31 @@ """Asynchronous worker for analyzing image submissions.""" +import os import threading from pathlib import Path from typing import Any +import sentry_sdk +from sentry_sdk.integrations.rq import RqIntegration +from sentry_sdk.integrations.threading import ThreadingIntegration +from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration + +# Initialize Sentry for workers (separate process from Flask app) +SENTRY_DSN = os.environ.get("SENTRY_DSN") +if SENTRY_DSN: + sentry_sdk.init( + dsn=SENTRY_DSN, + integrations=[ + RqIntegration(), + ThreadingIntegration(propagate_hub=True), # Capture exceptions in threads + SqlalchemyIntegration(), + ], + traces_sample_rate=float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")), + environment=os.environ.get("ENVIRONMENT", "development"), + release=os.environ.get("SENTRY_RELEASE", "1.0.0"), + send_default_pii=False, + ) + from .analyzers.binwalk import analyze_binwalk from .analyzers.decomposer import analyze_decomposer from .analyzers.exiftool import analyze_exiftool @@ -51,6 +73,10 @@ def analyze_image(submission_hash: str) -> None: image = Image.query.get_or_404(submission.image_hash) # type: ignore if not submission or not image: # No submission found + sentry_sdk.capture_message( + f"Submission or image not found: {submission_hash}", + level="warning" + ) return submission.status = "running" # type: ignore @@ -69,6 +95,7 @@ def analyze_image(submission_hash: str) -> None: analyzer_func(*args) except Exception as e: print(f"Error in {analyzer_func.__name__}: {e}") + sentry_sdk.capture_exception(e) analyzers = [ (analyze_binwalk, img_path, result_path), @@ -100,7 +127,10 @@ def analyze_image(submission_hash: str) -> None: thread.join() submission.status = "completed" - except Exception: + except Exception as e: + sentry_sdk.capture_exception(e) submission.status = "error" finally: db.session.commit() # pylint: disable=no-member + # Flush Sentry events - wait up to 5 seconds for events to be sent + sentry_sdk.flush(timeout=5) From 9d693b89125badb3b4faaf8e989b7887513f7ed0 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Tue, 30 Dec 2025 17:35:50 +0530 Subject: [PATCH 015/102] Add Sentry SDK to requirements for enhanced error tracking --- aperisolve/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aperisolve/requirements.txt b/aperisolve/requirements.txt index 14446ca..f7ee61b 100644 --- a/aperisolve/requirements.txt +++ b/aperisolve/requirements.txt @@ -7,4 +7,5 @@ psycopg2-binary types-Flask-SQLAlchemy sqlalchemy-stubs Pillow -gunicorn \ No newline at end of file +gunicorn +sentry-sdk[flask,sqlalchemy,rq] \ No newline at end of file From ff61763e577d717f1dcc36483a30abdbf90dc51a Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Tue, 30 Dec 2025 17:39:18 +0530 Subject: [PATCH 016/102] Add Threading integration to Sentry for improved error tracking --- aperisolve/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aperisolve/app.py b/aperisolve/app.py index 5b0ad4d..663ae07 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -12,6 +12,7 @@ import sentry_sdk from sentry_sdk.integrations.flask import FlaskIntegration from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration from sentry_sdk.integrations.rq import RqIntegration +from sentry_sdk.integrations.threading import ThreadingIntegration from flask import Flask, Response, abort, jsonify, render_template, request, send_file from redis import Redis @@ -31,6 +32,7 @@ if SENTRY_DSN: FlaskIntegration(), SqlalchemyIntegration(), RqIntegration(), + ThreadingIntegration(propagate_hub=True), ], # Set traces_sample_rate to 1.0 to capture 100% of transactions for performance monitoring. # Adjust this value in production (e.g., 0.1 for 10%) From b15dca9c537d418dba39567761b38198207ed5d1 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Tue, 30 Dec 2025 16:57:08 +0100 Subject: [PATCH 017/102] Fix with fs remove --- aperisolve/app.py | 80 ++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index 75359d2..f9e2ff9 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -6,6 +6,7 @@ import os import time from datetime import datetime, timezone from pathlib import Path +from shutil import rmtree from typing import Any, Optional from flask import Flask, Response, abort, jsonify, render_template, request, send_file @@ -112,34 +113,33 @@ def upload_image() -> tuple[Response, int]: submission_hash = hashlib.md5(submission_data).hexdigest() submission_path = RESULT_FOLDER / img_hash / submission_hash - # Check if hash is present on the DB - submission: Submission = Submission.query.filter_by( - hash=submission_hash - ).first() + # If submission already exists, return submission hash + if submission_path.exists(): + submission: Submission = Submission.query.filter_by( + hash=submission_hash + ).first() - # Only return if BOTH the file exists AND the DB entry exists - if submission_path.exists() and submission is not None: - return jsonify({"submission_hash": submission.hash}), 200 - + if submission is not None: # File exist but not in DB (should not happen), + rmtree(submission_path) # Removing file and continue processing as new + db.session.delete(submission) # pylint: disable=no-member + db.session.commit() # pylint: disable=no-member + else: + return jsonify({"submission_hash": submission.hash}), 200 - # Self-Healing Logic + # If image is new, create a new Image entry new_img_path = RESULT_FOLDER / img_hash / image_name + sub_img = Image.query.filter_by(hash=img_hash).first() # type: ignore - # Try to find image in DB by hash - sub_img = Image.query.filter_by( - hash=img_hash - ).first() + if new_img_path.parent.exists() and sub_img is None: + # Inconsistent state: image file exists but no DB entry + rmtree(new_img_path.parent) # Remove the existing files and continue - # If DB entry is missing (even if file exists on disk), create it! - if sub_img is None: - if not new_img_path.parent.exists() or not new_img_path.exists(): - new_img_path.parent.mkdir(parents=True, exist_ok=True) - with open(new_img_path, "wb") as f: - f.write(image_data) + if not new_img_path.parent.exists(): + new_img_path.parent.mkdir(parents=True, exist_ok=True) + with open(new_img_path, "wb") as f: # Write file to disk + f.write(image_data) - - # Recreate the missing Database Entry - sub_img = Image( + new_img = Image( file=str(new_img_path), hash=img_hash, size=len(image_data), @@ -147,39 +147,33 @@ def upload_image() -> tuple[Response, int]: first_submission_date=datetime.now(timezone.utc), last_submission_date=datetime.now(timezone.utc), ) - db.session.add(sub_img) - db.session.commit() + db.session.add(new_img) # pylint: disable=no-member + db.session.commit() # pylint: disable=no-member - # Increment upload count for the image + sub_img = Image.query.filter_by(hash=img_hash).first() # type: ignore sub_img.upload_count += 1 - db.session.commit() + db.session.commit() # pylint: disable=no-member # Create new Submission entry submission_path.mkdir(parents=True, exist_ok=True) + submission = Submission( + filename=image.filename, + password=password, + deep_analysis=deep_analysis, + hash=submission_hash, + status="pending", + date=time.time(), + image_hash=sub_img.hash, # type: ignore + ) + db.session.add(submission) # pylint: disable=no-member - # Only create a new submission if it does not already exist - # New submission for new image - if not submission: - submission = Submission( - filename=image.filename, - password=password, - deep_analysis=deep_analysis, - hash=submission_hash, - status="pending", - date=time.time(), - image_hash=sub_img.hash, - ) - db.session.add(submission) - db.session.commit() - + db.session.commit() # pylint: disable=no-member # Start the analysis jobs queue.enqueue("aperisolve.workers.analyze_image", submission.hash, job_timeout=300) - # Return submission identifier return jsonify({"submission_hash": submission.hash}), 200 - @app.route("/status/", methods=["GET"]) def get_status(hash_val: str) -> Response: """Get the processing status of a submission.""" From 64f13f53c395c8235c9c00a7993c33fd35e77b6c Mon Sep 17 00:00:00 2001 From: Zeecka Date: Tue, 30 Dec 2025 17:06:08 +0100 Subject: [PATCH 018/102] Fix Dockerfile --- Dockerfile | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1ac3310..1c0ce92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,14 +3,22 @@ FROM python:3.14-slim AS build WORKDIR / # Install build tools and dependencies +RUN apt-get update && apt-get install -y \ + wget \ + default-jdk \ + ruby \ + zip \ + 7zip - -RUN apt-get update && apt-get install -y --no-install-recommends \ - wget default-jre ruby \ - bzip2 xz-utils gzip lzip lzma lzop tar unzip p7zip-full squashfs-tools \ - file qpdf cpio arj cabextract sharutils lz4 ccache mtd-utils \ - python3-lxml upx-ucl zlib1g-dev liblzma-dev \ - binwalk foremost exiftool steghide binutils outguess pngcheck \ +# Install steganography and forensics tools +RUN apt-get update && apt-get install -y \ + binwalk \ + foremost \ + exiftool \ + steghide \ + binutils \ + outguess \ + pngcheck \ && gem install zsteg # Install OpenStego From 474ad6208588f1cf54300f2f30331cd7464763a5 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Tue, 30 Dec 2025 17:08:34 +0100 Subject: [PATCH 019/102] fix error showing binwalk --- aperisolve/analyzers/binwalk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aperisolve/analyzers/binwalk.py b/aperisolve/analyzers/binwalk.py index 823590c..950d14a 100644 --- a/aperisolve/analyzers/binwalk.py +++ b/aperisolve/analyzers/binwalk.py @@ -45,7 +45,7 @@ def analyze_binwalk(input_img: Path, output_dir: Path) -> None: if extracted_dir.exists(): shutil.rmtree(extracted_dir) - if len(stderr) > 0: + if len(stderr) > 0 and not zip_exist: err: dict[str, dict[str, Any]] = { "binwalk": { "status": "error", From 12382fee6ab9e4fcfc707f4622d1233f150ae4fd Mon Sep 17 00:00:00 2001 From: Zeecka Date: Tue, 30 Dec 2025 17:10:14 +0100 Subject: [PATCH 020/102] add comments --- aperisolve/analyzers/binwalk.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aperisolve/analyzers/binwalk.py b/aperisolve/analyzers/binwalk.py index 950d14a..6c05b9a 100644 --- a/aperisolve/analyzers/binwalk.py +++ b/aperisolve/analyzers/binwalk.py @@ -45,6 +45,8 @@ def analyze_binwalk(input_img: Path, output_dir: Path) -> None: if extracted_dir.exists(): shutil.rmtree(extracted_dir) + # Only report error if stderr exists AND extraction failed. + # If zip_exist is True, binwalk worked (despite warnings), so we proceed. if len(stderr) > 0 and not zip_exist: err: dict[str, dict[str, Any]] = { "binwalk": { From bb50df89262a3f45863014207e649ec901579326 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Wed, 31 Dec 2025 00:11:24 +0530 Subject: [PATCH 021/102] Refactor upload_image function to improve submission handling and self-healing logic --- aperisolve/app.py | 83 +++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 38 deletions(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index f9e2ff9..dd06e8d 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -6,7 +6,6 @@ import os import time from datetime import datetime, timezone from pathlib import Path -from shutil import rmtree from typing import Any, Optional from flask import Flask, Response, abort, jsonify, render_template, request, send_file @@ -113,33 +112,32 @@ def upload_image() -> tuple[Response, int]: submission_hash = hashlib.md5(submission_data).hexdigest() submission_path = RESULT_FOLDER / img_hash / submission_hash - # If submission already exists, return submission hash - if submission_path.exists(): - submission: Submission = Submission.query.filter_by( - hash=submission_hash - ).first() + # Check if hash is present on the DB + submission: Submission = Submission.query.filter_by( + hash=submission_hash + ).first() - if submission is not None: # File exist but not in DB (should not happen), - rmtree(submission_path) # Removing file and continue processing as new - db.session.delete(submission) # pylint: disable=no-member - db.session.commit() # pylint: disable=no-member - else: - return jsonify({"submission_hash": submission.hash}), 200 - - # If image is new, create a new Image entry + # Only return if BOTH the file exists AND the DB entry exists + if submission_path.exists() and submission is not None: + return jsonify({"submission_hash": submission.hash}), 200 + + # Self-Healing Logic new_img_path = RESULT_FOLDER / img_hash / image_name - sub_img = Image.query.filter_by(hash=img_hash).first() # type: ignore - if new_img_path.parent.exists() and sub_img is None: - # Inconsistent state: image file exists but no DB entry - rmtree(new_img_path.parent) # Remove the existing files and continue - - if not new_img_path.parent.exists(): + # This fixes the "Ghost File" issue (DB exists, but file is missing) + if not new_img_path.exists(): new_img_path.parent.mkdir(parents=True, exist_ok=True) - with open(new_img_path, "wb") as f: # Write file to disk + with open(new_img_path, "wb") as f: f.write(image_data) + + # Try to find image in DB by hash + sub_img = Image.query.filter_by( + hash=img_hash + ).first() - new_img = Image( + # If DB entry is missing (even if file exists on disk), create it! + if sub_img is None: + sub_img = Image( file=str(new_img_path), hash=img_hash, size=len(image_data), @@ -147,30 +145,39 @@ def upload_image() -> tuple[Response, int]: first_submission_date=datetime.now(timezone.utc), last_submission_date=datetime.now(timezone.utc), ) - db.session.add(new_img) # pylint: disable=no-member - db.session.commit() # pylint: disable=no-member - - sub_img = Image.query.filter_by(hash=img_hash).first() # type: ignore + db.session.add(sub_img) + db.session.commit() + + # Increment upload count for the image sub_img.upload_count += 1 - db.session.commit() # pylint: disable=no-member + db.session.commit() # Create new Submission entry submission_path.mkdir(parents=True, exist_ok=True) - submission = Submission( - filename=image.filename, - password=password, - deep_analysis=deep_analysis, - hash=submission_hash, - status="pending", - date=time.time(), - image_hash=sub_img.hash, # type: ignore - ) - db.session.add(submission) # pylint: disable=no-member - db.session.commit() # pylint: disable=no-member + # Only create a new submission if it does not already exist + # New submission for new image + if not submission: + submission = Submission( + filename=image.filename, + password=password, + deep_analysis=deep_analysis, + hash=submission_hash, + status="pending", + date=time.time(), + image_hash=sub_img.hash, + ) + db.session.add(submission) + db.session.commit() + + # Re-analysis case to prevent early return + submission.status = "pending" + db.session.commit() + # Start the analysis jobs queue.enqueue("aperisolve.workers.analyze_image", submission.hash, job_timeout=300) + # Return submission identifier return jsonify({"submission_hash": submission.hash}), 200 From 1923b43256a851b4ec4613970eab3bd93a2e5291 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 01:52:12 +0100 Subject: [PATCH 022/102] Lint --- aperisolve/app.py | 20 ++++++++++---------- aperisolve/workers.py | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index dd06e8d..2dc7164 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -120,7 +120,7 @@ def upload_image() -> tuple[Response, int]: # Only return if BOTH the file exists AND the DB entry exists if submission_path.exists() and submission is not None: return jsonify({"submission_hash": submission.hash}), 200 - + # Self-Healing Logic new_img_path = RESULT_FOLDER / img_hash / image_name @@ -129,7 +129,7 @@ def upload_image() -> tuple[Response, int]: new_img_path.parent.mkdir(parents=True, exist_ok=True) with open(new_img_path, "wb") as f: f.write(image_data) - + # Try to find image in DB by hash sub_img = Image.query.filter_by( hash=img_hash @@ -145,12 +145,12 @@ def upload_image() -> tuple[Response, int]: first_submission_date=datetime.now(timezone.utc), last_submission_date=datetime.now(timezone.utc), ) - db.session.add(sub_img) - db.session.commit() - + db.session.add(sub_img) # pylint: disable=no-member + db.session.commit() # pylint: disable=no-member + # Increment upload count for the image sub_img.upload_count += 1 - db.session.commit() + db.session.commit() # pylint: disable=no-member # Create new Submission entry submission_path.mkdir(parents=True, exist_ok=True) @@ -167,12 +167,12 @@ def upload_image() -> tuple[Response, int]: date=time.time(), image_hash=sub_img.hash, ) - db.session.add(submission) - db.session.commit() + db.session.add(submission) # pylint: disable=no-member + db.session.commit() # pylint: disable=no-member # Re-analysis case to prevent early return - submission.status = "pending" - db.session.commit() + submission.status = "pending" # type: ignore + db.session.commit() # pylint: disable=no-member # Start the analysis jobs queue.enqueue("aperisolve.workers.analyze_image", submission.hash, job_timeout=300) diff --git a/aperisolve/workers.py b/aperisolve/workers.py index c5be8c7..8542035 100644 --- a/aperisolve/workers.py +++ b/aperisolve/workers.py @@ -99,8 +99,8 @@ def analyze_image(submission_hash: str) -> None: for thread in threads: thread.join() - submission.status = "completed" + submission.status = "completed" # type: ignore except Exception: - submission.status = "error" + submission.status = "error" # type: ignore finally: db.session.commit() # pylint: disable=no-member From bf3974ba5ce9ad7b9fa09ebeae36bd0a82c193d6 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 01:53:24 +0100 Subject: [PATCH 023/102] sentry_sdk --- aperisolve/workers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aperisolve/workers.py b/aperisolve/workers.py index 8542035..b3c1903 100644 --- a/aperisolve/workers.py +++ b/aperisolve/workers.py @@ -68,6 +68,7 @@ def analyze_image(submission_hash: str) -> None: try: analyzer_func(*args) except Exception as e: + sentry_sdk.capture_exception(e) print(f"Error in {analyzer_func.__name__}: {e}") analyzers = [ From d3a36178154a866d9cd36efec1b3825bf53f882b Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 04:02:00 +0100 Subject: [PATCH 024/102] Sentry file, and header linting hints --- aperisolve/__init__.py | 3 ++ aperisolve/analyzers/__init__.py | 3 ++ aperisolve/analyzers/binwalk.py | 3 ++ aperisolve/analyzers/decomposer.py | 3 ++ aperisolve/analyzers/exiftool.py | 3 ++ aperisolve/analyzers/foremost.py | 3 ++ aperisolve/analyzers/image_resize.py | 3 ++ aperisolve/analyzers/openstego.py | 3 ++ aperisolve/analyzers/outguess.py | 3 ++ aperisolve/analyzers/pngcheck.py | 3 ++ aperisolve/analyzers/steghide.py | 3 ++ aperisolve/analyzers/strings.py | 3 ++ aperisolve/analyzers/template_analyzer.py | 3 ++ aperisolve/analyzers/utils.py | 3 ++ aperisolve/analyzers/zsteg.py | 3 ++ aperisolve/app.py | 39 ++++--------------- aperisolve/cleanup.py | 3 ++ aperisolve/config.py | 3 ++ aperisolve/models.py | 3 ++ aperisolve/sentry.py | 46 +++++++++++++++++++++++ aperisolve/workers.py | 34 +++++------------ aperisolve/wsgi.py | 3 ++ 22 files changed, 120 insertions(+), 56 deletions(-) create mode 100644 aperisolve/sentry.py diff --git a/aperisolve/__init__.py b/aperisolve/__init__.py index e69de29..538f764 100644 --- a/aperisolve/__init__.py +++ b/aperisolve/__init__.py @@ -0,0 +1,3 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable diff --git a/aperisolve/analyzers/__init__.py b/aperisolve/analyzers/__init__.py index e69de29..538f764 100644 --- a/aperisolve/analyzers/__init__.py +++ b/aperisolve/analyzers/__init__.py @@ -0,0 +1,3 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable diff --git a/aperisolve/analyzers/binwalk.py b/aperisolve/analyzers/binwalk.py index 6c05b9a..9b8cfac 100644 --- a/aperisolve/analyzers/binwalk.py +++ b/aperisolve/analyzers/binwalk.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Binwalk Analyzer for Image Submissions.""" import shutil diff --git a/aperisolve/analyzers/decomposer.py b/aperisolve/analyzers/decomposer.py index e6440d3..cff14d5 100644 --- a/aperisolve/analyzers/decomposer.py +++ b/aperisolve/analyzers/decomposer.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Bits Decomposer Analyzer for Image Submissions.""" from pathlib import Path diff --git a/aperisolve/analyzers/exiftool.py b/aperisolve/analyzers/exiftool.py index e13e2e9..5ed107d 100644 --- a/aperisolve/analyzers/exiftool.py +++ b/aperisolve/analyzers/exiftool.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Exiftool Analyzer for Image Submissions.""" import subprocess diff --git a/aperisolve/analyzers/foremost.py b/aperisolve/analyzers/foremost.py index 5c55972..eb0fc8c 100644 --- a/aperisolve/analyzers/foremost.py +++ b/aperisolve/analyzers/foremost.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Foremost Analyzer for Image Submissions.""" import shutil diff --git a/aperisolve/analyzers/image_resize.py b/aperisolve/analyzers/image_resize.py index bf30897..17db9b5 100644 --- a/aperisolve/analyzers/image_resize.py +++ b/aperisolve/analyzers/image_resize.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Image size bruteforcer for common PNG sizes.""" from pathlib import Path diff --git a/aperisolve/analyzers/openstego.py b/aperisolve/analyzers/openstego.py index 597356a..aa6e989 100644 --- a/aperisolve/analyzers/openstego.py +++ b/aperisolve/analyzers/openstego.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """OpenStego Analyzer for Image Submissions.""" import shutil diff --git a/aperisolve/analyzers/outguess.py b/aperisolve/analyzers/outguess.py index 976979b..9fc6a78 100644 --- a/aperisolve/analyzers/outguess.py +++ b/aperisolve/analyzers/outguess.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Outguess Analyzer for Image Submissions.""" import shutil diff --git a/aperisolve/analyzers/pngcheck.py b/aperisolve/analyzers/pngcheck.py index a398bd5..730e67c 100644 --- a/aperisolve/analyzers/pngcheck.py +++ b/aperisolve/analyzers/pngcheck.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Pngcheck Analyzer for Image Submissions.""" import subprocess diff --git a/aperisolve/analyzers/steghide.py b/aperisolve/analyzers/steghide.py index d74d077..0602bd6 100644 --- a/aperisolve/analyzers/steghide.py +++ b/aperisolve/analyzers/steghide.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Steghide Analyzer for Image Submissions.""" import re diff --git a/aperisolve/analyzers/strings.py b/aperisolve/analyzers/strings.py index 42c0d53..6757fdb 100644 --- a/aperisolve/analyzers/strings.py +++ b/aperisolve/analyzers/strings.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Strings Analyzer for Image Submissions.""" import subprocess diff --git a/aperisolve/analyzers/template_analyzer.py b/aperisolve/analyzers/template_analyzer.py index bf86779..0008497 100644 --- a/aperisolve/analyzers/template_analyzer.py +++ b/aperisolve/analyzers/template_analyzer.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Custom Analyzer for Image Submissions.""" import shutil diff --git a/aperisolve/analyzers/utils.py b/aperisolve/analyzers/utils.py index da86987..3387106 100644 --- a/aperisolve/analyzers/utils.py +++ b/aperisolve/analyzers/utils.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Utility functions for analyzers modules.""" import fcntl diff --git a/aperisolve/analyzers/zsteg.py b/aperisolve/analyzers/zsteg.py index f530390..0435841 100644 --- a/aperisolve/analyzers/zsteg.py +++ b/aperisolve/analyzers/zsteg.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Zsteg Analyzer for Image Submissions.""" import subprocess diff --git a/aperisolve/app.py b/aperisolve/app.py index a3215b2..3b1ee5d 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Aperi'Solve Flask application.""" import hashlib @@ -9,46 +12,18 @@ from pathlib import Path from typing import Any, Optional import sentry_sdk -from sentry_sdk.integrations.flask import FlaskIntegration -from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration -from sentry_sdk.integrations.rq import RqIntegration -from sentry_sdk.integrations.threading import ThreadingIntegration - from flask import Flask, Response, abort, jsonify, render_template, request, send_file from redis import Redis from rq import Queue +from .sentry import initialize_sentry + +initialize_sentry() + from .cleanup import cleanup_old_entries from .config import IMAGE_EXTENSIONS, RESULT_FOLDER, WORKER_FILES from .models import Image, Submission, db, init_db -SENTRY_DSN = os.environ.get("SENTRY_DSN") -ENVIRONMENT = os.environ.get("ENVIRONMENT", "development") - -if SENTRY_DSN: - sentry_sdk.init( - dsn=SENTRY_DSN, - integrations=[ - FlaskIntegration(), - SqlalchemyIntegration(), - RqIntegration(), - ThreadingIntegration(propagate_hub=True), - ], - # Set traces_sample_rate to 1.0 to capture 100% of transactions for performance monitoring. - # Adjust this value in production (e.g., 0.1 for 10%) - traces_sample_rate=float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")), - # Set profiles_sample_rate to 1.0 to profile 100% of sampled transactions. - profiles_sample_rate=float(os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1")), - # Environment tag (production, staging, development) - environment=ENVIRONMENT, - # Release version (use git commit hash or version number) - release=os.environ.get("SENTRY_RELEASE", "1.0.0"), - # Send default PII (user IPs, etc.) - set to False for privacy - send_default_pii=False, - # Enable automatic breadcrumb tracking - enable_tracing=True, - ) - app: Flask = Flask(__name__) app.json.sort_keys = False # type: ignore app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("DB_URI") diff --git a/aperisolve/cleanup.py b/aperisolve/cleanup.py index 402cd25..5a63f96 100644 --- a/aperisolve/cleanup.py +++ b/aperisolve/cleanup.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Cleanup old entries in the database and file system.""" import shutil diff --git a/aperisolve/config.py b/aperisolve/config.py index 96016c3..f4a6458 100644 --- a/aperisolve/config.py +++ b/aperisolve/config.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Aperi'Solve configuration variables.""" from pathlib import Path diff --git a/aperisolve/models.py b/aperisolve/models.py index fa0d345..b064689 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """This module defines the database models for the Aperi'Solve application.""" import itertools diff --git a/aperisolve/sentry.py b/aperisolve/sentry.py new file mode 100644 index 0000000..bb87683 --- /dev/null +++ b/aperisolve/sentry.py @@ -0,0 +1,46 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable +"""Sentry initialization for Aperisolve.""" + +import os + +import sentry_sdk +from sentry_sdk.integrations.flask import FlaskIntegration +from sentry_sdk.integrations.rq import RqIntegration +from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration +from sentry_sdk.integrations.threading import ThreadingIntegration + +SENTRY_DSN = os.environ.get("SENTRY_DSN") +ENVIRONMENT = os.environ.get("ENVIRONMENT", "development") + + +def initialize_sentry() -> None: + """Initialize Sentry SDK once, safely.""" + + # No DSN → do nothing + dsn = os.environ.get("SENTRY_DSN") + if not dsn: + return + + # Prevent double initialization (Flask + RQ, CLI, tests, etc.) + if sentry_sdk.Hub.current.client is not None: + return + + sentry_sdk.init( + dsn=dsn, + integrations=[ + FlaskIntegration(), + SqlalchemyIntegration(), + RqIntegration(), + ThreadingIntegration(propagate_hub=True), + ], + traces_sample_rate=float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")), + profiles_sample_rate=float( + os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1") + ), + environment=os.environ.get("ENVIRONMENT", "development"), + release=os.environ.get("SENTRY_RELEASE", "1.0.0"), + send_default_pii=False, + enable_tracing=True, + ) diff --git a/aperisolve/workers.py b/aperisolve/workers.py index 247b40b..72b7861 100644 --- a/aperisolve/workers.py +++ b/aperisolve/workers.py @@ -1,32 +1,19 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Asynchronous worker for analyzing image submissions.""" -import os import threading from pathlib import Path from typing import Any import sentry_sdk -from sentry_sdk.integrations.rq import RqIntegration -from sentry_sdk.integrations.threading import ThreadingIntegration -from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration -# Initialize Sentry for workers (separate process from Flask app) -SENTRY_DSN = os.environ.get("SENTRY_DSN") -if SENTRY_DSN: - sentry_sdk.init( - dsn=SENTRY_DSN, - integrations=[ - RqIntegration(), - ThreadingIntegration(propagate_hub=True), # Capture exceptions in threads - SqlalchemyIntegration(), - ], - traces_sample_rate=float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")), - environment=os.environ.get("ENVIRONMENT", "development"), - release=os.environ.get("SENTRY_RELEASE", "1.0.0"), - send_default_pii=False, - ) +from .sentry import initialize_sentry -from .analyzers.binwalk import analyze_binwalk +initialize_sentry() + +from .analyzers.binwalk import analyze_binwalk # pylint: disable=C0413 from .analyzers.decomposer import analyze_decomposer from .analyzers.exiftool import analyze_exiftool from .analyzers.foremost import analyze_foremost @@ -74,8 +61,7 @@ def analyze_image(submission_hash: str) -> None: if not submission or not image: # No submission found sentry_sdk.capture_message( - f"Submission or image not found: {submission_hash}", - level="warning" + f"Submission or image not found: {submission_hash}", level="warning" ) return @@ -127,10 +113,10 @@ def analyze_image(submission_hash: str) -> None: for thread in threads: thread.join() - submission.status = "completed" + submission.status = "completed" # type: ignore except Exception as e: sentry_sdk.capture_exception(e) - submission.status = "error" + submission.status = "error" # type: ignore finally: db.session.commit() # pylint: disable=no-member # Flush Sentry events - wait up to 5 seconds for events to be sent diff --git a/aperisolve/wsgi.py b/aperisolve/wsgi.py index 0bf784b..a220be7 100644 --- a/aperisolve/wsgi.py +++ b/aperisolve/wsgi.py @@ -1,3 +1,6 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,R0903,R0801 +# mypy: disable-error-code=unused-awaitable """Aperi'Solve WSGI runner.""" from .app import app as application From f83e6e571ed230295e21039d245bf24d95e8bae2 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 04:41:57 +0100 Subject: [PATCH 025/102] Fix 110 --- README.md | 27 +++++++++++++++++++++++---- compose.dev.yml | 18 +++++++++++------- compose.yml | 22 ++++++++++++---------- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 929dbc2..115dbae 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Support Aperi'Solve: Aperi'Solve is an open-source steganalysis web platform that performs automated analysis on images to detect and extract hidden data using common steganography tools and techniques. Table of contents + - [Key features](#key-features) - [Quick start (Docker)](#quick-start-docker) - [Development](#development) @@ -59,6 +60,7 @@ In case you want to host your own version of https://www.aperisolve.com/. Recommended: Docker + Docker Compose. Clone and start the full stack (production-like): + ```bash git clone https://github.com/Zeecka/AperiSolve.git cd AperiSolve @@ -68,13 +70,16 @@ docker compose up -d Default: http://localhost:5000/ ## Development + Development compose: + ```bash # development environment (hot reload and local volumes) docker compose -f compose.dev.yml up --build ``` Useful commands: + ```bash # Stop and remove containers, networks and volumes (results are stored in a volume) docker compose down -v @@ -84,21 +89,26 @@ docker exec -it aperisolve-web bash # Enter Postgres shell (from host) docker exec -it postgres psql -U aperiuser -d aperisolve + +# Backup all uploaded files +docker cp -r aperisolve-web:/aperisolve/results /path/to/backup/location + +# Backup a single uploaded file +docker cp aperisolve-web:/aperisolve/results/filename.ext /path/to/backup/filename.ext ``` > [!WARNING] -> If switching between dev and production compose files, remove the `results` directory or mounted volume to avoid conflicts: -> ```bash -> rm -rf aperisolve/results -> ``` +> Uploaded files are stored in a tmpfs. This means that they are stored in memory and will be lost if the server is restarted. Ensure that you back up any important files before the server is stopped or restarted. ### Adding a new analyzer Adding a custom analyzer is straightforward: 1. Create your analyzer file: + - Copy `aperisolve/analyzers/template_analyzer.py` -> `aperisolve/analyzers/myanalyzer.py` - Implement function signature similar to: + ```python # aperisolve/analyzers/myanalyzer.py def analyze_myanalyzer(image_path: str, results_dir: str) -> dict: @@ -111,7 +121,9 @@ Adding a custom analyzer is straightforward: ``` 2. Register the analyzer in the worker pipeline: + - Edit `aperisolve/workers.py`: + ```python # import from .analyzers.myanalyzer import analyze_myanalyzer @@ -126,23 +138,27 @@ Adding a custom analyzer is straightforward: ``` 3. Add your analyzer to the UI order: + - Edit `aperisolve/static/js/aperisolve.js` and append `myanalyzer` to `TOOL_ORDER` so it appears in the frontend. 4. Test locally: run the worker and submit jobs to ensure outputs are produced and displayed. > [!TIP] +> > - Keep analyzers idempotent and write outputs to the provided `results_dir`. > - Return structured JSON so the frontend can render links/downloads automatically. ## Configuration & environment variables Typical services: + - Web app (Flask) - Workers (Python) - Redis (RQ) - PostgreSQL Main environment variables (examples): + - DATABASE_URL=postgresql://aperiuser:password@postgres:5432/aperisolve - REDIS_URL=redis://redis:6379/0 - SECRET_KEY=change_me @@ -185,12 +201,14 @@ This separation keeps heavy tools (binwalk, foremost, zsteg, etc.) isolated and ## Contributing Contributions are welcome! Please: + 1. Fork the repository 2. Create a feature branch 3. Open a pull request describing your change > [!IMPORTANT] > Please follow the code style and run linters before submitting. The project adheres to: +> > - Black > - Flake8 (ignoring E203, E501, W503) > - Pylint (ignoring W0718, R0903, R0801) @@ -206,4 +224,5 @@ CI will run these checks on each PR. ## Credits Acknowledgements: + - Thanks to contributors and the open-source community for the tools integrated ([zsteg](https://github.com/zed-0xff/zsteg), [steghide](https://steghide.sourceforge.net/), [outguess](https://www.rbcafe.com/software/outguess/), [openstego](https://www.openstego.com/), [exiftool](https://exiftool.org/), [binwalk](https://github.com/ReFirmLabs/binwalk), [foremost](https://foremost.sourceforge.net/), [pngcheck](https://www.libpng.org/pub/png/apps/pngcheck.html), [strings](https://pubs.opengroup.org/onlinepubs/9799919799/utilities/strings.html), ...). diff --git a/compose.dev.yml b/compose.dev.yml index e23163d..ce877d3 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -9,6 +9,9 @@ services: - "5000:5000" volumes: - ./aperisolve:/aperisolve + # Commented tmpfs for development to allow persistent results during code changes + tmpfs: + - /aperisolve/results depends_on: postgres: condition: service_healthy @@ -30,6 +33,9 @@ services: command: rq worker default --url redis://redis:6379/0 volumes: - ./aperisolve:/aperisolve + # Commented tmpfs for development to allow persistent results during code changes + tmpfs: + - /aperisolve/results depends_on: - redis - postgres @@ -40,18 +46,19 @@ services: - FLASK_ENV=development redis: - image: redis:7 + image: redis:8 container_name: redis restart: always env_file: - .env postgres: - image: postgres:14 + image: postgres:18 container_name: postgres restart: always - volumes: - - pgdata:/var/lib/postgresql/data + # Commented tmpfs for development to allow persistent results during code changes + tmpfs: + - /var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s @@ -71,6 +78,3 @@ services: - redis env_file: - .env - -volumes: - pgdata: \ No newline at end of file diff --git a/compose.yml b/compose.yml index 777fef2..612016d 100644 --- a/compose.yml +++ b/compose.yml @@ -6,8 +6,8 @@ services: restart: always ports: - "5000:5000" - volumes: - - apresults:/aperisolve/results + tmpfs: + - /aperisolve/results depends_on: postgres: condition: service_healthy @@ -17,6 +17,12 @@ services: condition: service_started env_file: - .env + healthcheck: + test: ["CMD-SHELL", "dd if=/dev/urandom of=/tmp/healthcheck.png bs=1 count=1000000 && curl -X POST -F 'image=@/tmp/healthcheck.png' http://localhost:5000/upload -s -o /dev/null -w '%{http_code}' | grep -q 200"] + interval: 3600s + timeout: 30s + retries: 3 + start_period: 900s worker: image: ghcr.io/zeecka/aperisolve:latest @@ -24,8 +30,8 @@ services: container_name: aperisolve-worker restart: always command: rq worker default --url redis://redis:6379/0 - volumes: - - apresults:/aperisolve/results + tmpfs: + - /aperisolve/results depends_on: - redis - postgres @@ -36,8 +42,8 @@ services: image: postgres:14 container_name: postgres restart: always - volumes: - - pgdata:/var/lib/postgresql/data + tmpfs: + - /var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s @@ -63,7 +69,3 @@ services: - redis env_file: - .env - -volumes: - apresults: - pgdata: From 2f50a23b8ccbcd35b09a0699708cbd2742178e63 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 04:44:14 +0100 Subject: [PATCH 026/102] change readme roadmap --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 115dbae..48fa04c 100644 --- a/README.md +++ b/README.md @@ -187,11 +187,7 @@ This separation keeps heavy tools (binwalk, foremost, zsteg, etc.) isolated and ## Roadmap -- [ ] **[Bug]** Error 500 when file already exist on disc - [ ] **[Bug]** Duplicate key value violates unique constraint "submission_pkey" -- [ ] **[Bug]** Binwalk Extractor error -- [ ] **[Feature]** Implement healthcheck and auto reboot -- [ ] **[Feature]** Implement Sentry for better error handling - [ ] **[Feature]** Zsteg: full extraction (--all) and download of discovered files (mp3, etc.) - [ ] **[Feature]** Mobile-friendly UI / test on mobile - [ ] **[Feature]** i18n (internationalization) From 29cdeaea4c2e150be67d433c38e8c35410f62be8 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 04:45:07 +0100 Subject: [PATCH 027/102] format, black --- aperisolve/app.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index 3b1ee5d..295faaa 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -43,10 +43,13 @@ queue = Queue(connection=redis_conn) @app.errorhandler(413) def too_large(_: Any) -> tuple[Response, int]: """Error Handler for Max File Size.""" - sentry_sdk.set_context("upload", { - "content_length": request.content_length, - "max_allowed": app.config["MAX_CONTENT_LENGTH"], - }) + sentry_sdk.set_context( + "upload", + { + "content_length": request.content_length, + "max_allowed": app.config["MAX_CONTENT_LENGTH"], + }, + ) sentry_sdk.capture_message("Upload failed: 413 Payload Too Large", level="warning") return jsonify({"error": "Image size exceeded"}), 413 @@ -56,8 +59,10 @@ def not_found(_: Any) -> str: """Error Handler for 404 not found.""" return render_template("error.html", message="Resource not found", statuscode=404) + # Sentry automatically captures 500 errors + @app.route("/") def index() -> str: """Render the main index page.""" @@ -127,9 +132,7 @@ def upload_image() -> tuple[Response, int]: submission_path = RESULT_FOLDER / img_hash / submission_hash # Check if hash is present on the DB - submission: Submission = Submission.query.filter_by( - hash=submission_hash - ).first() + submission: Submission = Submission.query.filter_by(hash=submission_hash).first() # Only return if BOTH the file exists AND the DB entry exists if submission_path.exists() and submission is not None: @@ -145,9 +148,7 @@ def upload_image() -> tuple[Response, int]: f.write(image_data) # Try to find image in DB by hash - sub_img = Image.query.filter_by( - hash=img_hash - ).first() + sub_img = Image.query.filter_by(hash=img_hash).first() # If DB entry is missing (even if file exists on disk), create it! if sub_img is None: From fc02dcab80844b620de2edbde1bdacd544dda584 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 04:53:19 +0100 Subject: [PATCH 028/102] bump postgres_14 --- compose.dev.yml | 2 +- compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose.dev.yml b/compose.dev.yml index ce877d3..df6e5c2 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -58,7 +58,7 @@ services: restart: always # Commented tmpfs for development to allow persistent results during code changes tmpfs: - - /var/lib/postgresql/data + - /var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s diff --git a/compose.yml b/compose.yml index 612016d..1191cae 100644 --- a/compose.yml +++ b/compose.yml @@ -43,7 +43,7 @@ services: container_name: postgres restart: always tmpfs: - - /var/lib/postgresql/data + - /var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s From ddce01ec8d3a49c5c106eb0eae62526f673c18ce Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 05:28:58 +0100 Subject: [PATCH 029/102] change docker entrypoint to clear content --- Dockerfile | 2 +- aperisolve/static/js/aperisolve.js | 4 +++- compose.dev.yml | 25 +++++++++++++------------ compose.yml | 11 ++++++++--- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1c0ce92..cff09e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,4 +42,4 @@ RUN apt-get remove -y wget && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "--access-logfile", "-", "--error-logfile", "-", "--log-level", "info", "--capture-output", "aperisolve.wsgi:application"] +CMD ["sh", "-c", "rm -rf /aperisolve/results/* && gunicorn -w 4 -b 0.0.0.0:5000 --access-logfile - --error-logfile - --log-level info --capture-output aperisolve.wsgi:application"] diff --git a/aperisolve/static/js/aperisolve.js b/aperisolve/static/js/aperisolve.js index 0090c17..0294ead 100644 --- a/aperisolve/static/js/aperisolve.js +++ b/aperisolve/static/js/aperisolve.js @@ -486,7 +486,9 @@ if (browseBtn) { } catch (e) { showDanger("❌ Invalid server response.", true); } - } else { + } else if (xhr.status == 413) { + showDanger("❌ File too large. Please upload a smaller file.", true); + }else { showDanger(`❌ HTTP error ${xhr.status}`, true); } progressBar.style.width = "100%"; diff --git a/compose.dev.yml b/compose.dev.yml index df6e5c2..9df9dea 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -4,14 +4,13 @@ services: build: . container_name: aperisolve-web restart: always - command: ["flask", "--app", "aperisolve.app", "run", "--host=0.0.0.0", "--port=5000"] + command: ["/bin/sh", "-c", "rm -rf /aperisolve/results/* && flask --app aperisolve.app run --host=0.0.0.0 --port=5000"] + #command: ["/bin/sh", "-c", "flask --app aperisolve.app run --host=0.0.0.0 --port=5000"] ports: - "5000:5000" volumes: - ./aperisolve:/aperisolve - # Commented tmpfs for development to allow persistent results during code changes - tmpfs: - - /aperisolve/results + - apresults:/aperisolve/results # Shared named volume (renamed) depends_on: postgres: condition: service_healthy @@ -30,12 +29,11 @@ services: build: . container_name: aperisolve-worker restart: always - command: rq worker default --url redis://redis:6379/0 + command: ["/bin/sh", "-c", "rm -rf /aperisolve/results/* && rq worker default --url redis://redis:6379/0"] + #command: ["/bin/sh", "-c", "rq worker default --url redis://redis:6379/0"] volumes: - ./aperisolve:/aperisolve - # Commented tmpfs for development to allow persistent results during code changes - tmpfs: - - /aperisolve/results + - apresults:/aperisolve/results # Shared named volume (renamed) depends_on: - redis - postgres @@ -56,9 +54,9 @@ services: image: postgres:18 container_name: postgres restart: always - # Commented tmpfs for development to allow persistent results during code changes - tmpfs: - - /var/lib/postgresql + entrypoint: ["/bin/sh", "-c", "rm -rf /var/lib/postgresql/* && docker-entrypoint.sh postgres"] + volumes: + - pgdata:/var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s @@ -67,7 +65,6 @@ services: env_file: - .env - rqdashboard: image: eoranged/rq-dashboard container_name: rq-dashboard @@ -78,3 +75,7 @@ services: - redis env_file: - .env + +volumes: + apresults: + pgdata: diff --git a/compose.yml b/compose.yml index 1191cae..afa902f 100644 --- a/compose.yml +++ b/compose.yml @@ -39,11 +39,12 @@ services: - .env postgres: - image: postgres:14 + image: postgres:18 container_name: postgres restart: always - tmpfs: - - /var/lib/postgresql + entrypoint: ["/bin/sh", "-c", "rm -rf /var/lib/postgresql/* && docker-entrypoint.sh postgres"] + volumes: + - pgdata:/var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s @@ -69,3 +70,7 @@ services: - redis env_file: - .env + +volumes: + apresults: + pgdata: From cbd80203c9241f6667223bc611d6f299e81156ea Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 05:32:02 +0100 Subject: [PATCH 030/102] lint black --- aperisolve/app.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index 3b1ee5d..295faaa 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -43,10 +43,13 @@ queue = Queue(connection=redis_conn) @app.errorhandler(413) def too_large(_: Any) -> tuple[Response, int]: """Error Handler for Max File Size.""" - sentry_sdk.set_context("upload", { - "content_length": request.content_length, - "max_allowed": app.config["MAX_CONTENT_LENGTH"], - }) + sentry_sdk.set_context( + "upload", + { + "content_length": request.content_length, + "max_allowed": app.config["MAX_CONTENT_LENGTH"], + }, + ) sentry_sdk.capture_message("Upload failed: 413 Payload Too Large", level="warning") return jsonify({"error": "Image size exceeded"}), 413 @@ -56,8 +59,10 @@ def not_found(_: Any) -> str: """Error Handler for 404 not found.""" return render_template("error.html", message="Resource not found", statuscode=404) + # Sentry automatically captures 500 errors + @app.route("/") def index() -> str: """Render the main index page.""" @@ -127,9 +132,7 @@ def upload_image() -> tuple[Response, int]: submission_path = RESULT_FOLDER / img_hash / submission_hash # Check if hash is present on the DB - submission: Submission = Submission.query.filter_by( - hash=submission_hash - ).first() + submission: Submission = Submission.query.filter_by(hash=submission_hash).first() # Only return if BOTH the file exists AND the DB entry exists if submission_path.exists() and submission is not None: @@ -145,9 +148,7 @@ def upload_image() -> tuple[Response, int]: f.write(image_data) # Try to find image in DB by hash - sub_img = Image.query.filter_by( - hash=img_hash - ).first() + sub_img = Image.query.filter_by(hash=img_hash).first() # If DB entry is missing (even if file exists on disk), create it! if sub_img is None: From 9c37efb403915efa81b5ca42f454b19e7d1b96f8 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 15:58:51 +0100 Subject: [PATCH 031/102] opti sentry --- aperisolve/sentry.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/aperisolve/sentry.py b/aperisolve/sentry.py index bb87683..e0677a4 100644 --- a/aperisolve/sentry.py +++ b/aperisolve/sentry.py @@ -12,35 +12,31 @@ from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration from sentry_sdk.integrations.threading import ThreadingIntegration SENTRY_DSN = os.environ.get("SENTRY_DSN") -ENVIRONMENT = os.environ.get("ENVIRONMENT", "development") - +SENTRY_ENVIRONMENT = os.environ.get("SENTRY_ENVIRONMENT", "development") +SENTRY_TRACES_SAMPLE_RATE = float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")) +SENTRY_PROFILES_SAMPLE_RATE = float(os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1")) +SENTRY_RELEASE = os.environ.get("SENTRY_RELEASE", "1.0.0") def initialize_sentry() -> None: """Initialize Sentry SDK once, safely.""" # No DSN → do nothing - dsn = os.environ.get("SENTRY_DSN") - if not dsn: - return - # Prevent double initialization (Flask + RQ, CLI, tests, etc.) - if sentry_sdk.Hub.current.client is not None: + if not SENTRY_DSN or sentry_sdk.Hub.current.client is not None: return sentry_sdk.init( - dsn=dsn, + dsn=SENTRY_DSN, integrations=[ FlaskIntegration(), SqlalchemyIntegration(), RqIntegration(), ThreadingIntegration(propagate_hub=True), ], - traces_sample_rate=float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")), - profiles_sample_rate=float( - os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1") - ), - environment=os.environ.get("ENVIRONMENT", "development"), - release=os.environ.get("SENTRY_RELEASE", "1.0.0"), + traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE, + profiles_sample_rate=SENTRY_PROFILES_SAMPLE_RATE, + environment=SENTRY_ENVIRONMENT, + release=SENTRY_RELEASE, send_default_pii=False, enable_tracing=True, ) From 58edd548355d0428f6a8ec077f64e49801cbb4eb Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 15:59:13 +0100 Subject: [PATCH 032/102] remove cleanup in compose and docker --- Dockerfile | 2 +- compose.dev.yml | 9 +++------ compose.yml | 9 ++++----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index cff09e7..1c0ce92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,4 +42,4 @@ RUN apt-get remove -y wget && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -CMD ["sh", "-c", "rm -rf /aperisolve/results/* && gunicorn -w 4 -b 0.0.0.0:5000 --access-logfile - --error-logfile - --log-level info --capture-output aperisolve.wsgi:application"] +CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "--access-logfile", "-", "--error-logfile", "-", "--log-level", "info", "--capture-output", "aperisolve.wsgi:application"] diff --git a/compose.dev.yml b/compose.dev.yml index 9df9dea..c10943f 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -4,13 +4,12 @@ services: build: . container_name: aperisolve-web restart: always - command: ["/bin/sh", "-c", "rm -rf /aperisolve/results/* && flask --app aperisolve.app run --host=0.0.0.0 --port=5000"] - #command: ["/bin/sh", "-c", "flask --app aperisolve.app run --host=0.0.0.0 --port=5000"] + command: ["/bin/sh", "-c", "flask --app aperisolve.app run --host=0.0.0.0 --port=5000"] ports: - "5000:5000" volumes: - ./aperisolve:/aperisolve - - apresults:/aperisolve/results # Shared named volume (renamed) + - apresults:/aperisolve/results depends_on: postgres: condition: service_healthy @@ -29,8 +28,7 @@ services: build: . container_name: aperisolve-worker restart: always - command: ["/bin/sh", "-c", "rm -rf /aperisolve/results/* && rq worker default --url redis://redis:6379/0"] - #command: ["/bin/sh", "-c", "rq worker default --url redis://redis:6379/0"] + command: ["/bin/sh", "-c", "rq worker default --url redis://redis:6379/0"] volumes: - ./aperisolve:/aperisolve - apresults:/aperisolve/results # Shared named volume (renamed) @@ -54,7 +52,6 @@ services: image: postgres:18 container_name: postgres restart: always - entrypoint: ["/bin/sh", "-c", "rm -rf /var/lib/postgresql/* && docker-entrypoint.sh postgres"] volumes: - pgdata:/var/lib/postgresql healthcheck: diff --git a/compose.yml b/compose.yml index afa902f..2afcb73 100644 --- a/compose.yml +++ b/compose.yml @@ -6,8 +6,8 @@ services: restart: always ports: - "5000:5000" - tmpfs: - - /aperisolve/results + volumes: + - apresults:/aperisolve/results depends_on: postgres: condition: service_healthy @@ -30,8 +30,8 @@ services: container_name: aperisolve-worker restart: always command: rq worker default --url redis://redis:6379/0 - tmpfs: - - /aperisolve/results + volumes: + - apresults:/aperisolve/results depends_on: - redis - postgres @@ -42,7 +42,6 @@ services: image: postgres:18 container_name: postgres restart: always - entrypoint: ["/bin/sh", "-c", "rm -rf /var/lib/postgresql/* && docker-entrypoint.sh postgres"] volumes: - pgdata:/var/lib/postgresql healthcheck: From 3a9cf21c24341c7e331b3bfae0aa25f44195f629 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 16:00:43 +0100 Subject: [PATCH 033/102] fix sentry env and add comments --- .env | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 4116ee9..549b813 100644 --- a/.env +++ b/.env @@ -1,15 +1,24 @@ +# Application configuration MAX_CONTENT_LENGTH=1048576 MAX_PENDING_TIME=600 MAX_STORE_TIME=259200 + +# Flask configuration FLASK_DEBUG=0 FLASK_ENV=production -DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve + +# Redis configuration RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0 + +# Database configuration POSTGRES_DB=aperisolve POSTGRES_USER=aperiuser POSTGRES_PASSWORD=aperipass +DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve + +# Sentry configuration SENTRY_DSN=https://your-dsn@sentry.io/project-id SENTRY_TRACES_SAMPLE_RATE=0.1 SENTRY_PROFILES_SAMPLE_RATE=0.1 SENTRY_RELEASE=1.0.0 -ENVIRONMENT=production \ No newline at end of file +SENTRY_ENVIRONMENT=production From 5c5e63dbea949bd43758b8ada040f67a98ce2cb5 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 16:25:54 +0100 Subject: [PATCH 034/102] SKIP IHDR and Clear at restart --- .env | 2 ++ aperisolve/models.py | 18 ++++++++++++++---- compose.dev.yml | 1 + 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 549b813..366a17d 100644 --- a/.env +++ b/.env @@ -2,6 +2,8 @@ MAX_CONTENT_LENGTH=1048576 MAX_PENDING_TIME=600 MAX_STORE_TIME=259200 +CLEAR_AT_RESTART=1 +SKIP_IHDR_FILL=1 # Flask configuration FLASK_DEBUG=0 diff --git a/aperisolve/models.py b/aperisolve/models.py index b064689..6b300f8 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -5,6 +5,9 @@ import itertools from datetime import datetime, timezone +from os import getenv +from pathlib import Path +from shutil import rmtree from flask import Flask from flask_sqlalchemy import SQLAlchemy @@ -128,9 +131,16 @@ def init_db(app: Flask) -> None: """ with app.app_context(): if not db.engine.dialect.has_table(db.engine.connect(), "image"): - print("Creating database...") + print(f"CLEAR_AT_RESTART={getenv('CLEAR_AT_RESTART', '0')}") + if getenv("CLEAR_AT_RESTART", "0") == "1": + print("Clearing database and file system at restart...") + db.drop_all() # Clear existing tables if configured + rmtree(Path("./results"), ignore_errors=True) # Clear results folder db.create_all() print("Database structure created successfully.") - print("Filling IHDR lookup table...") - create_crc_db() - print("IHDR table filled successfully.") + if getenv("SKIP_IHDR_FILL", "0") == "1": + print("Skipping IHDR lookup table fill as per configuration.") + else: + print("Filling IHDR lookup table...") + create_crc_db() + print("IHDR table filled successfully.") diff --git a/compose.dev.yml b/compose.dev.yml index c10943f..523ee2f 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -22,6 +22,7 @@ services: environment: - FLASK_DEBUG=true - FLASK_ENV=development + - SKIP_IHDR_FILL=1 worker: #image: ghcr.io/zeecka/aperisolve:latest From 531bdc9d816ba076f964b8738eef165017e9f3df Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 16:26:10 +0100 Subject: [PATCH 035/102] Sentry error handling if DSN fail --- aperisolve/sentry.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/aperisolve/sentry.py b/aperisolve/sentry.py index e0677a4..22042f1 100644 --- a/aperisolve/sentry.py +++ b/aperisolve/sentry.py @@ -25,18 +25,23 @@ def initialize_sentry() -> None: if not SENTRY_DSN or sentry_sdk.Hub.current.client is not None: return - sentry_sdk.init( - dsn=SENTRY_DSN, - integrations=[ - FlaskIntegration(), - SqlalchemyIntegration(), - RqIntegration(), - ThreadingIntegration(propagate_hub=True), - ], - traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE, - profiles_sample_rate=SENTRY_PROFILES_SAMPLE_RATE, - environment=SENTRY_ENVIRONMENT, - release=SENTRY_RELEASE, - send_default_pii=False, - enable_tracing=True, - ) + try: + sentry_sdk.init( + dsn=SENTRY_DSN, + integrations=[ + FlaskIntegration(), + SqlalchemyIntegration(), + RqIntegration(), + ThreadingIntegration(propagate_hub=True), + ], + traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE, + profiles_sample_rate=SENTRY_PROFILES_SAMPLE_RATE, + environment=SENTRY_ENVIRONMENT, + release=SENTRY_RELEASE, + send_default_pii=False, + enable_tracing=True, + ) + except sentry_sdk.utils.BadDsn as e: + print(f"Sentry DSN is invalid, skipping Sentry initialization: {e}") + except Exception as e: + print(f"Sentry initialization failed: {e}") From 7e1290c5d92fb33af2dc34bd53cdab5371d988c4 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 16:30:12 +0100 Subject: [PATCH 036/102] work still in progress --- aperisolve/models.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/aperisolve/models.py b/aperisolve/models.py index 6b300f8..9d82e8c 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -130,12 +130,13 @@ def init_db(app: Flask) -> None: - Prints status messages to console during initialization """ with app.app_context(): - if not db.engine.dialect.has_table(db.engine.connect(), "image"): - print(f"CLEAR_AT_RESTART={getenv('CLEAR_AT_RESTART', '0')}") - if getenv("CLEAR_AT_RESTART", "0") == "1": - print("Clearing database and file system at restart...") - db.drop_all() # Clear existing tables if configured - rmtree(Path("./results"), ignore_errors=True) # Clear results folder + if getenv("CLEAR_AT_RESTART", "0") == "1": # Force clear if CLEAR_AT_RESTART + print("Clearing database and file system at restart...") + db.drop_all() # Clear existing tables if configured + db.session.commit() # pylint: disable=no-member + rmtree(Path("./results"), ignore_errors=True) # Clear results folder + + if not db.engine.dialect.has_table(db.engine.connect(), "image"): # If not configured db.create_all() print("Database structure created successfully.") if getenv("SKIP_IHDR_FILL", "0") == "1": From ad9501a3b2dd2f56d9efc123766edaed17d71c41 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 20:49:24 +0100 Subject: [PATCH 037/102] Lint black --- aperisolve/models.py | 4 +++- aperisolve/sentry.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/aperisolve/models.py b/aperisolve/models.py index 9d82e8c..c38ad96 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -136,7 +136,9 @@ def init_db(app: Flask) -> None: db.session.commit() # pylint: disable=no-member rmtree(Path("./results"), ignore_errors=True) # Clear results folder - if not db.engine.dialect.has_table(db.engine.connect(), "image"): # If not configured + if not db.engine.dialect.has_table( + db.engine.connect(), "image" + ): # If not configured db.create_all() print("Database structure created successfully.") if getenv("SKIP_IHDR_FILL", "0") == "1": diff --git a/aperisolve/sentry.py b/aperisolve/sentry.py index 22042f1..13b5871 100644 --- a/aperisolve/sentry.py +++ b/aperisolve/sentry.py @@ -14,9 +14,12 @@ from sentry_sdk.integrations.threading import ThreadingIntegration SENTRY_DSN = os.environ.get("SENTRY_DSN") SENTRY_ENVIRONMENT = os.environ.get("SENTRY_ENVIRONMENT", "development") SENTRY_TRACES_SAMPLE_RATE = float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")) -SENTRY_PROFILES_SAMPLE_RATE = float(os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1")) +SENTRY_PROFILES_SAMPLE_RATE = float( + os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1") +) SENTRY_RELEASE = os.environ.get("SENTRY_RELEASE", "1.0.0") + def initialize_sentry() -> None: """Initialize Sentry SDK once, safely.""" From 0031b7ec627342d8483b334e370e945fe4727e49 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Thu, 1 Jan 2026 02:48:22 +0530 Subject: [PATCH 038/102] chore: stop tracking .env, add to gitignore --- .env | 15 --------------- .gitignore | 1 - 2 files changed, 16 deletions(-) delete mode 100644 .env diff --git a/.env b/.env deleted file mode 100644 index 4116ee9..0000000 --- a/.env +++ /dev/null @@ -1,15 +0,0 @@ -MAX_CONTENT_LENGTH=1048576 -MAX_PENDING_TIME=600 -MAX_STORE_TIME=259200 -FLASK_DEBUG=0 -FLASK_ENV=production -DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve -RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0 -POSTGRES_DB=aperisolve -POSTGRES_USER=aperiuser -POSTGRES_PASSWORD=aperipass -SENTRY_DSN=https://your-dsn@sentry.io/project-id -SENTRY_TRACES_SAMPLE_RATE=0.1 -SENTRY_PROFILES_SAMPLE_RATE=0.1 -SENTRY_RELEASE=1.0.0 -ENVIRONMENT=production \ No newline at end of file diff --git a/.gitignore b/.gitignore index 5482ea4..646de31 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,3 @@ __pycache__/ .mypy_cache/ results/ -.env From 49ad16f02da9a1b5638ff5d7d99f5c8ccb772186 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Thu, 1 Jan 2026 02:49:25 +0530 Subject: [PATCH 039/102] chore: add .env to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 646de31..8a70dc2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ # Python +.env __pycache__/ .mypy_cache/ results/ From 676f655f0deb637cb8549c8cb9a5f83baf9ad6bb Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Thu, 1 Jan 2026 02:54:50 +0530 Subject: [PATCH 040/102] chore: add example environment configuration file --- .env.example | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b4bc151 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +MAX_CONTENT_LENGTH=1048576 +MAX_PENDING_TIME=600 +MAX_STORE_TIME=259200 +FLASK_DEBUG=0 +FLASK_ENV=development +DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve +RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0 +POSTGRES_DB=aperisolve +POSTGRES_USER=aperiuser +POSTGRES_PASSWORD=aperipass +SENTRY_DSN=https://your-dsn@sentry.io/project-id +SENTRY_TRACES_SAMPLE_RATE=0.1 +SENTRY_PROFILES_SAMPLE_RATE=0.1 +SENTRY_RELEASE=1.0.0 +ENVIRONMENT=development \ No newline at end of file From 52b084321c53202a298c42e5aa9d134002bfbf15 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Thu, 1 Jan 2026 02:55:09 +0530 Subject: [PATCH 041/102] chore: update env_file references to use .env.example in compose.dev.yml --- compose.dev.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/compose.dev.yml b/compose.dev.yml index e23163d..0b19a14 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -17,7 +17,7 @@ services: worker: condition: service_started env_file: - - .env + - .env.example environment: - FLASK_DEBUG=true - FLASK_ENV=development @@ -34,7 +34,7 @@ services: - redis - postgres env_file: - - .env + - .env.example environment: - FLASK_DEBUG=true - FLASK_ENV=development @@ -44,7 +44,7 @@ services: container_name: redis restart: always env_file: - - .env + - .env.example postgres: image: postgres:14 @@ -58,7 +58,7 @@ services: timeout: 5s retries: 5 env_file: - - .env + - .env.example rqdashboard: @@ -70,7 +70,7 @@ services: depends_on: - redis env_file: - - .env + - .env.example volumes: pgdata: \ No newline at end of file From 60bed74a0de06275c2720f38f53438d2b46a7530 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 22:47:20 +0100 Subject: [PATCH 042/102] Now working --- aperisolve/app.py | 5 +++-- aperisolve/models.py | 26 ++++++++++++-------------- compose.dev.yml | 1 + compose.yml | 2 ++ 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index 295faaa..734097c 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -33,9 +33,10 @@ app.config["MAX_CONTENT_LENGTH"] = int( ) RESULT_FOLDER.mkdir(parents=True, exist_ok=True) - db.init_app(app) -init_db(app) +with app.app_context(): + init_db(app) + redis_conn = Redis(host="redis", port=6379) queue = Queue(connection=redis_conn) diff --git a/aperisolve/models.py b/aperisolve/models.py index c38ad96..dc47053 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -62,7 +62,7 @@ class IHDR(db.Model): # type: ignore packed = Column(BigInteger, nullable=False) -def create_crc_db() -> None: +def fill_ihdr_db() -> None: """ Creates a database of common PNG IHDR (Image Header) entries by generating combinations of standard PNG parameters such as resolutions, bit depths, @@ -132,18 +132,16 @@ def init_db(app: Flask) -> None: with app.app_context(): if getenv("CLEAR_AT_RESTART", "0") == "1": # Force clear if CLEAR_AT_RESTART print("Clearing database and file system at restart...") - db.drop_all() # Clear existing tables if configured - db.session.commit() # pylint: disable=no-member + db.session.remove() # pylint: disable=no-member + db.drop_all() rmtree(Path("./results"), ignore_errors=True) # Clear results folder - if not db.engine.dialect.has_table( - db.engine.connect(), "image" - ): # If not configured - db.create_all() - print("Database structure created successfully.") - if getenv("SKIP_IHDR_FILL", "0") == "1": - print("Skipping IHDR lookup table fill as per configuration.") - else: - print("Filling IHDR lookup table...") - create_crc_db() - print("IHDR table filled successfully.") + db.create_all() + print("Database structure created successfully.") + + if getenv("SKIP_IHDR_FILL", "0") == "1": + print("Skipping IHDR lookup table fill as per configuration.") + else: + print("Filling IHDR lookup table...") + fill_ihdr_db() + print("IHDR table filled successfully.") diff --git a/compose.dev.yml b/compose.dev.yml index 523ee2f..1bdd982 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -41,6 +41,7 @@ services: environment: - FLASK_DEBUG=true - FLASK_ENV=development + - CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing redis: image: redis:8 diff --git a/compose.yml b/compose.yml index 2afcb73..4489e66 100644 --- a/compose.yml +++ b/compose.yml @@ -37,6 +37,8 @@ services: - postgres env_file: - .env + environment: + - CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing postgres: image: postgres:18 From e525fd0774e3a242c7cb1bfb37aad0b5f5c79f52 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 22:55:58 +0100 Subject: [PATCH 043/102] Set mounted volume back --- README.md | 5 ++++- compose.dev.yml | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 48fa04c..285ca5f 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,10 @@ docker cp aperisolve-web:/aperisolve/results/filename.ext /path/to/backup/filena ``` > [!WARNING] -> Uploaded files are stored in a tmpfs. This means that they are stored in memory and will be lost if the server is restarted. Ensure that you back up any important files before the server is stopped or restarted. +> If switching between dev and production compose files, remove the `results` directory or mounted volume to avoid conflicts: +> ```bash +> rm -rf aperisolve/results +> ``` ### Adding a new analyzer diff --git a/compose.dev.yml b/compose.dev.yml index 1bdd982..134bea6 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -9,7 +9,6 @@ services: - "5000:5000" volumes: - ./aperisolve:/aperisolve - - apresults:/aperisolve/results depends_on: postgres: condition: service_healthy @@ -32,7 +31,6 @@ services: command: ["/bin/sh", "-c", "rq worker default --url redis://redis:6379/0"] volumes: - ./aperisolve:/aperisolve - - apresults:/aperisolve/results # Shared named volume (renamed) depends_on: - redis - postgres From faec51c6c6e0209aad64a06b59b9cec7924b8c49 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 23:04:10 +0100 Subject: [PATCH 044/102] Rebase edits from main --- .env.example | 15 +++++++++++++-- compose.dev.yml | 5 ++--- compose.yml | 14 +++++++++++--- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index b4bc151..38a8106 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,26 @@ +# Application configuration MAX_CONTENT_LENGTH=1048576 MAX_PENDING_TIME=600 MAX_STORE_TIME=259200 +CLEAR_AT_RESTART=1 +SKIP_IHDR_FILL=1 + +# Flask configuration FLASK_DEBUG=0 FLASK_ENV=development -DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve + +# Redis configuration RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0 + +# Database configuration POSTGRES_DB=aperisolve POSTGRES_USER=aperiuser POSTGRES_PASSWORD=aperipass +DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve + +# Sentry configuration SENTRY_DSN=https://your-dsn@sentry.io/project-id SENTRY_TRACES_SAMPLE_RATE=0.1 SENTRY_PROFILES_SAMPLE_RATE=0.1 SENTRY_RELEASE=1.0.0 -ENVIRONMENT=development \ No newline at end of file +SENTRY_ENVIRONMENT=development diff --git a/compose.dev.yml b/compose.dev.yml index 0b19a14..6f46315 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -60,7 +60,6 @@ services: env_file: - .env.example - rqdashboard: image: eoranged/rq-dashboard container_name: rq-dashboard @@ -71,6 +70,6 @@ services: - redis env_file: - .env.example - + volumes: - pgdata: \ No newline at end of file + pgdata: diff --git a/compose.yml b/compose.yml index 777fef2..4489e66 100644 --- a/compose.yml +++ b/compose.yml @@ -17,6 +17,12 @@ services: condition: service_started env_file: - .env + healthcheck: + test: ["CMD-SHELL", "dd if=/dev/urandom of=/tmp/healthcheck.png bs=1 count=1000000 && curl -X POST -F 'image=@/tmp/healthcheck.png' http://localhost:5000/upload -s -o /dev/null -w '%{http_code}' | grep -q 200"] + interval: 3600s + timeout: 30s + retries: 3 + start_period: 900s worker: image: ghcr.io/zeecka/aperisolve:latest @@ -31,13 +37,15 @@ services: - postgres env_file: - .env + environment: + - CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing postgres: - image: postgres:14 + image: postgres:18 container_name: postgres restart: always volumes: - - pgdata:/var/lib/postgresql/data + - pgdata:/var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s @@ -63,7 +71,7 @@ services: - redis env_file: - .env - + volumes: apresults: pgdata: From 6f62eefd7787af3b4c81e2e5f39a2c59a9590ca0 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 23:07:15 +0100 Subject: [PATCH 045/102] black lint --- aperisolve/app.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/aperisolve/app.py b/aperisolve/app.py index 3b1ee5d..295faaa 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -43,10 +43,13 @@ queue = Queue(connection=redis_conn) @app.errorhandler(413) def too_large(_: Any) -> tuple[Response, int]: """Error Handler for Max File Size.""" - sentry_sdk.set_context("upload", { - "content_length": request.content_length, - "max_allowed": app.config["MAX_CONTENT_LENGTH"], - }) + sentry_sdk.set_context( + "upload", + { + "content_length": request.content_length, + "max_allowed": app.config["MAX_CONTENT_LENGTH"], + }, + ) sentry_sdk.capture_message("Upload failed: 413 Payload Too Large", level="warning") return jsonify({"error": "Image size exceeded"}), 413 @@ -56,8 +59,10 @@ def not_found(_: Any) -> str: """Error Handler for 404 not found.""" return render_template("error.html", message="Resource not found", statuscode=404) + # Sentry automatically captures 500 errors + @app.route("/") def index() -> str: """Render the main index page.""" @@ -127,9 +132,7 @@ def upload_image() -> tuple[Response, int]: submission_path = RESULT_FOLDER / img_hash / submission_hash # Check if hash is present on the DB - submission: Submission = Submission.query.filter_by( - hash=submission_hash - ).first() + submission: Submission = Submission.query.filter_by(hash=submission_hash).first() # Only return if BOTH the file exists AND the DB entry exists if submission_path.exists() and submission is not None: @@ -145,9 +148,7 @@ def upload_image() -> tuple[Response, int]: f.write(image_data) # Try to find image in DB by hash - sub_img = Image.query.filter_by( - hash=img_hash - ).first() + sub_img = Image.query.filter_by(hash=img_hash).first() # If DB entry is missing (even if file exists on disk), create it! if sub_img is None: From 6f2c412864663ee43d9597fba0a617e6c5c9d830 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 23:15:13 +0100 Subject: [PATCH 046/102] remove old .env file --- .env | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 .env diff --git a/.env b/.env deleted file mode 100644 index 366a17d..0000000 --- a/.env +++ /dev/null @@ -1,26 +0,0 @@ -# Application configuration -MAX_CONTENT_LENGTH=1048576 -MAX_PENDING_TIME=600 -MAX_STORE_TIME=259200 -CLEAR_AT_RESTART=1 -SKIP_IHDR_FILL=1 - -# Flask configuration -FLASK_DEBUG=0 -FLASK_ENV=production - -# Redis configuration -RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0 - -# Database configuration -POSTGRES_DB=aperisolve -POSTGRES_USER=aperiuser -POSTGRES_PASSWORD=aperipass -DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve - -# Sentry configuration -SENTRY_DSN=https://your-dsn@sentry.io/project-id -SENTRY_TRACES_SAMPLE_RATE=0.1 -SENTRY_PROFILES_SAMPLE_RATE=0.1 -SENTRY_RELEASE=1.0.0 -SENTRY_ENVIRONMENT=production From cac572dec9d030e6f4caf36d2dca75f39742df39 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Wed, 31 Dec 2025 23:16:31 +0100 Subject: [PATCH 047/102] fix conflict compose dev --- compose.dev.yml | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/compose.dev.yml b/compose.dev.yml index 134bea6..6f46315 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -4,7 +4,7 @@ services: build: . container_name: aperisolve-web restart: always - command: ["/bin/sh", "-c", "flask --app aperisolve.app run --host=0.0.0.0 --port=5000"] + command: ["flask", "--app", "aperisolve.app", "run", "--host=0.0.0.0", "--port=5000"] ports: - "5000:5000" volumes: @@ -17,50 +17,48 @@ services: worker: condition: service_started env_file: - - .env + - .env.example environment: - FLASK_DEBUG=true - FLASK_ENV=development - - SKIP_IHDR_FILL=1 worker: #image: ghcr.io/zeecka/aperisolve:latest build: . container_name: aperisolve-worker restart: always - command: ["/bin/sh", "-c", "rq worker default --url redis://redis:6379/0"] + command: rq worker default --url redis://redis:6379/0 volumes: - ./aperisolve:/aperisolve depends_on: - redis - postgres env_file: - - .env + - .env.example environment: - FLASK_DEBUG=true - FLASK_ENV=development - - CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing redis: - image: redis:8 + image: redis:7 container_name: redis restart: always env_file: - - .env + - .env.example postgres: - image: postgres:18 + image: postgres:14 container_name: postgres restart: always volumes: - - pgdata:/var/lib/postgresql + - pgdata:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U aperiuser -d aperisolve"] interval: 5s timeout: 5s retries: 5 env_file: - - .env + - .env.example rqdashboard: image: eoranged/rq-dashboard @@ -71,8 +69,7 @@ services: depends_on: - redis env_file: - - .env + - .env.example volumes: - apresults: pgdata: From c4f9d36f58054b60e7a60bc09a4d4e19fde8a9ca Mon Sep 17 00:00:00 2001 From: Zeecka Date: Thu, 1 Jan 2026 17:41:56 +0100 Subject: [PATCH 048/102] add isort and black line-length --- .pre-commit-config.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 16a1f1f..b8fa7af 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,12 @@ repos: rev: 25.11.0 hooks: - id: black + args: ['--line-length', '100'] +- repo: https://github.com/pycqa/isort + rev: 5.11.2 + hooks: + - id: isort + args: ['--profile', 'black'] - repo: https://github.com/PyCQA/flake8 rev: 7.3.0 hooks: @@ -25,4 +31,4 @@ repos: hooks: - id: mypy args: ["--disallow-any-generics", "--disallow-untyped-def", "--follow-imports", "skip", "--ignore-missing-imports", "--disable-error-code", "unused-awaitable"] - language: system \ No newline at end of file + language: system From 1dee160d7b3f751211a89f2f728cc018bf9a45f0 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Thu, 1 Jan 2026 17:42:06 +0100 Subject: [PATCH 049/102] add doc about isort --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 285ca5f..b84a9ea 100644 --- a/README.md +++ b/README.md @@ -208,10 +208,13 @@ Contributions are welcome! Please: > [!IMPORTANT] > Please follow the code style and run linters before submitting. The project adheres to: > -> - Black +> - Black (line length 100) +> - Isort (profile black) > - Flake8 (ignoring E203, E501, W503) > - Pylint (ignoring W0718, R0903, R0801) > - Mypy (ignoring unused-awaitable) +> +> See [.pre-commit-config.yaml](.pre-commit-config.yaml) file. CI will run these checks on each PR. From 4ef3e2db59444e6f4fc5e301b14e2215ab8a4149 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Thu, 1 Jan 2026 17:42:28 +0100 Subject: [PATCH 050/102] init base_analyzer --- aperisolve/analyzers/base_analyzer.py | 133 ++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 aperisolve/analyzers/base_analyzer.py diff --git a/aperisolve/analyzers/base_analyzer.py b/aperisolve/analyzers/base_analyzer.py new file mode 100644 index 0000000..ac2814b --- /dev/null +++ b/aperisolve/analyzers/base_analyzer.py @@ -0,0 +1,133 @@ +# flake8: noqa: E203,E501,W503 +# pylint: disable=C0413,W0718,W0613,R0903,R0801 +# mypy: disable-error-code=unused-awaitable +"""Base Analyzer class for image analysis tools.""" + +import subprocess +from abc import ABC +from pathlib import Path +from shutil import rmtree +from typing import Any, Optional, overload + +from .utils import MAX_PENDING_TIME, update_data + + +class SubprocessAnalyzer(ABC): + """Analyzer that runs a subprocess command.""" + + name: str + input_img: Path + output_dir: Path + has_archive: bool + cmd: list[str] | None = None + img: str + + def __init__(self, name: str, input_img: Path, output_dir: Path, has_archive: bool = False): + self.name = name + self.input_img = input_img + self.img = f"../{self.input_img.name}" + self.output_dir = output_dir + self.has_archive = has_archive + + def run_command( + self, cmd: list[str], cwd: Optional[Path] = None + ) -> subprocess.CompletedProcess[str]: + """Run a subprocess command.""" + return subprocess.run( + cmd, + cwd=cwd, + capture_output=True, + text=True, + check=False, + timeout=MAX_PENDING_TIME, + ) + + def generate_archive(self, output_dir: Path, extracted_dir: Path) -> str: + """Zip the extracted files and remove the directory.""" + zip_data = self.run_command(["7z", "a", f"../{self.name}.7z", "*"], cwd=extracted_dir) + rmtree(extracted_dir) + return zip_data.stderr + + def update_result(self, result: dict[str, Any]) -> None: + """Update the results.json with the analysis result.""" + update_data(self.output_dir, {self.name: result}) + + def get_extracted_dir(self) -> Path: + """Get the extracted directory path. Can be overridden but work as it is.""" + return self.output_dir / self.name + + @overload + def build_cmd(self) -> list[str]: ... + + @overload + def build_cmd(self, password: str) -> list[str]: ... + + def build_cmd(self, password: Optional[str] = None) -> list[str]: + """Build the command to run. Can be overridden.""" + if self.cmd is None: + raise NotImplementedError("cmd must be set or build_cmd overridden") + return self.cmd + + @overload + def analyze(self) -> None: ... + + @overload + def analyze(self, password: str) -> None: ... + + def analyze(self, password: Optional[str] = None) -> None: + """Run the subprocess command and handle results.""" + result: dict[str, Any] + try: + extracted_dir = None + if self.has_archive: + extracted_dir = self.get_extracted_dir() + extracted_dir.mkdir(parents=True, exist_ok=True) + + if password: + cmd = self.build_cmd(password) + else: + cmd = self.build_cmd() + data = self.run_command(cmd, cwd=self.output_dir) + returncode = data.returncode + stderr = data.stderr + stdout = data.stdout + + zip_exist = False + if extracted_dir and extracted_dir.exists() and any(extracted_dir.iterdir()): + self.generate_archive(self.output_dir, extracted_dir) # keep archive + zip_exist = True + + if self.is_error(returncode, stdout, stderr, zip_exist): + result = { + "status": "error", + } + result["error"] = self.process_error(stdout, stderr) + else: + result = { + "status": "ok", + "output": self.process_output(stdout, stderr), + } + note = self.process_note(stdout, stderr) + if note: + result["note"] = note + if zip_exist: + result["download"] = f"/download/{self.output_dir.name}/{self.name}" + self.update_result(result) + except Exception as e: + self.update_result({"status": "error", "error": str(e)}) + + def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: + """Check if the result is an error.""" + return len(stderr) > 0 + + def process_output(self, stdout: str, stderr: str) -> str | list[str] | dict[str, str]: + """Process the stdout into a list of lines.""" + return [line for line in stdout.split("\n") if line] if stdout else [] + + def process_error(self, stdout: str, stderr: str) -> str: + """Process the stderr.""" + return stderr + + def process_note(self, stdout: str, stderr: str) -> str: + """Process the stdout for informational purposes.""" + return stdout From a44ec01ce13413d22e8d4c7b18016aaad4235bba Mon Sep 17 00:00:00 2001 From: Zeecka Date: Thu, 1 Jan 2026 21:56:06 +0100 Subject: [PATCH 051/102] CLEAR_AT_RESTART=0 for compose dev --- compose.dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.dev.yml b/compose.dev.yml index 6f46315..77baee1 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -38,6 +38,7 @@ services: environment: - FLASK_DEBUG=true - FLASK_ENV=development + - CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing redis: image: redis:7 From e400f657867bf4d8e6b1c5eda6903b320173307c Mon Sep 17 00:00:00 2001 From: Zeecka Date: Thu, 1 Jan 2026 23:30:54 +0100 Subject: [PATCH 052/102] update base analyser --- aperisolve/analyzers/base_analyzer.py | 51 +++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/aperisolve/analyzers/base_analyzer.py b/aperisolve/analyzers/base_analyzer.py index ac2814b..6d3748c 100644 --- a/aperisolve/analyzers/base_analyzer.py +++ b/aperisolve/analyzers/base_analyzer.py @@ -3,13 +3,18 @@ # mypy: disable-error-code=unused-awaitable """Base Analyzer class for image analysis tools.""" +import fcntl +import json +import os import subprocess +import threading from abc import ABC from pathlib import Path from shutil import rmtree from typing import Any, Optional, overload -from .utils import MAX_PENDING_TIME, update_data +_thread_lock = threading.Lock() +MAX_PENDING_TIME = int(os.getenv("MAX_PENDING_TIME", "600")) # 10 minutes by default class SubprocessAnalyzer(ABC): @@ -21,6 +26,7 @@ class SubprocessAnalyzer(ABC): has_archive: bool cmd: list[str] | None = None img: str + make_folder: bool = True def __init__(self, name: str, input_img: Path, output_dir: Path, has_archive: bool = False): self.name = name @@ -49,8 +55,38 @@ class SubprocessAnalyzer(ABC): return zip_data.stderr def update_result(self, result: dict[str, Any]) -> None: - """Update the results.json with the analysis result.""" - update_data(self.output_dir, {self.name: result}) + """Thread-safe and process-safe JSON update using lock file and atomic replace.""" + json_file = self.output_dir / "results.json" + new_data = {self.name: result} + lock_file = json_file.with_suffix(json_file.suffix + ".lock") + tmp_file = json_file.with_suffix(json_file.suffix + ".tmp") + + json_file.parent.mkdir(parents=True, exist_ok=True) + + with _thread_lock: # synchronizes across threads + with open(lock_file, "w", encoding="utf-8") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) # synchronizes across processes + + try: + # Read existing JSON + data: dict[Any, Any] = {} + if json_file.exists(): + try: + with open(json_file, "r", encoding="utf-8") as f: + data = json.load(f) + except json.JSONDecodeError: + data = {} + + # Update with new data + data.update(new_data) + + # Write safely to a temp file + with open(tmp_file, "w", encoding="utf-8") as f: + json.dump(data, f, sort_keys=False) + + os.replace(tmp_file, json_file) # ensures file write is atomic + finally: + fcntl.flock(lock, fcntl.LOCK_UN) def get_extracted_dir(self) -> Path: """Get the extracted directory path. Can be overridden but work as it is.""" @@ -81,7 +117,8 @@ class SubprocessAnalyzer(ABC): extracted_dir = None if self.has_archive: extracted_dir = self.get_extracted_dir() - extracted_dir.mkdir(parents=True, exist_ok=True) + if self.make_folder: + extracted_dir.mkdir(parents=True, exist_ok=True) if password: cmd = self.build_cmd(password) @@ -107,7 +144,7 @@ class SubprocessAnalyzer(ABC): "status": "ok", "output": self.process_output(stdout, stderr), } - note = self.process_note(stdout, stderr) + note: Optional[str] = self.process_note(stdout, stderr) if note: result["note"] = note if zip_exist: @@ -128,6 +165,6 @@ class SubprocessAnalyzer(ABC): """Process the stderr.""" return stderr - def process_note(self, stdout: str, stderr: str) -> str: + def process_note(self, stdout: str, stderr: str) -> Optional[str]: """Process the stdout for informational purposes.""" - return stdout + return None From 78f8d867ea8f72486709286645aedc0f04a585ec Mon Sep 17 00:00:00 2001 From: Zeecka Date: Thu, 1 Jan 2026 23:31:42 +0100 Subject: [PATCH 053/102] Update analysers (WIP) --- aperisolve/analyzers/binwalk.py | 84 +++--------- aperisolve/analyzers/exiftool.py | 57 +++----- aperisolve/analyzers/foremost.py | 72 +++------- aperisolve/analyzers/outguess.py | 103 +++------------ aperisolve/analyzers/pngcheck.py | 59 ++------- aperisolve/analyzers/steghide.py | 154 +++++++--------------- aperisolve/analyzers/strings.py | 47 ++----- aperisolve/analyzers/template_analyzer.py | 85 +++--------- aperisolve/analyzers/zsteg.py | 54 +++----- 9 files changed, 189 insertions(+), 526 deletions(-) diff --git a/aperisolve/analyzers/binwalk.py b/aperisolve/analyzers/binwalk.py index 9b8cfac..eb04845 100644 --- a/aperisolve/analyzers/binwalk.py +++ b/aperisolve/analyzers/binwalk.py @@ -3,75 +3,27 @@ # mypy: disable-error-code=unused-awaitable """Binwalk Analyzer for Image Submissions.""" -import shutil -import subprocess from pathlib import Path -from typing import Any -from .utils import MAX_PENDING_TIME, update_data +from .base_analyzer import SubprocessAnalyzer + + +class BinwalkAnalyzer(SubprocessAnalyzer): + """Analyzer for binwalk.""" + + def __init__(self, *args): + super().__init__("binwalk", *args, has_archive=True) + self.cmd = ["binwalk", "--matryoshka", "-e", self.img, "--run-as=root"] + self.make_folder = False # Binwalk already create its own folder + + def get_extracted_dir(self) -> Path: + return self.output_dir / f"_{self.input_img.name}.extracted" + + def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: + return len(stderr) > 0 and not zip_exist and False def analyze_binwalk(input_img: Path, output_dir: Path) -> None: """Analyze an image submission using binwalk.""" - - image_name = input_img.name - extracted_dir = output_dir / f"_{image_name}.extracted" - - try: - stderr = "" - # Run binwalk - data = subprocess.run( - ["binwalk", "--matryoshka", "-e", "../" + str(image_name), "--run-as=root"], - cwd=output_dir, - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - stderr += data.stderr - - zip_exist = False - if extracted_dir.exists(): - # Zip extracted files - zip_data = subprocess.run( - ["7z", "a", "../binwalk.7z", "*"], - cwd=extracted_dir, - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - zip_exist = True - stderr += zip_data.stderr - - # Remove the extracted directory - if extracted_dir.exists(): - shutil.rmtree(extracted_dir) - - # Only report error if stderr exists AND extraction failed. - # If zip_exist is True, binwalk worked (despite warnings), so we proceed. - if len(stderr) > 0 and not zip_exist: - err: dict[str, dict[str, Any]] = { - "binwalk": { - "status": "error", - "output": data.stdout.split("\n") if data else [], - "error": stderr, - } - } - update_data(output_dir, err) - return None - - output_data: dict[str, dict[str, Any]] = { - "binwalk": { - "status": "ok", - "output": data.stdout.split("\n") if data else [], - } - } - if zip_exist: - output_data["binwalk"]["download"] = f"/download/{output_dir.name}/binwalk" - - update_data(output_dir, output_data) - - except Exception as e: - update_data(output_dir, {"binwalk": {"status": "error", "error": str(e)}}) - return None + analyzer = BinwalkAnalyzer(input_img, output_dir) + analyzer.analyze() diff --git a/aperisolve/analyzers/exiftool.py b/aperisolve/analyzers/exiftool.py index 5ed107d..ae42bae 100644 --- a/aperisolve/analyzers/exiftool.py +++ b/aperisolve/analyzers/exiftool.py @@ -3,46 +3,29 @@ # mypy: disable-error-code=unused-awaitable """Exiftool Analyzer for Image Submissions.""" -import subprocess from pathlib import Path -from .utils import MAX_PENDING_TIME, update_data +from .base_analyzer import SubprocessAnalyzer + + +class ExiftoolAnalyzer(SubprocessAnalyzer): + """Analyzer for exiftool.""" + + def __init__(self, *args): + super().__init__("exiftool", *args) + self.cmd = ["exiftool", "-a", "-u", "-g1", self.img] + + def process_output(self, stdout: str, stderr: str) -> dict[str, str]: + """Process the stdout into a list of lines.""" + metadata: dict[str, str] = {} + for line in stdout.split("\n"): + if ":" in line: + key, value = line.split(":", 1) + metadata[key.strip()] = value.strip() + return metadata def analyze_exiftool(input_img: Path, output_dir: Path) -> None: """Analyze an image submission using exiftool.""" - - try: - data = subprocess.run( - ["exiftool", input_img], - capture_output=True, - text=True, - check=True, - timeout=MAX_PENDING_TIME, - ) - - metadata: dict[str, str] = {} - for line in data.stdout.split("\n"): - if ":" in line: - key, value = line.split(":", 1) - metadata[key.strip()] = value.strip() - - if data.stderr: - update_data( - output_dir, - { - "exiftool": { - "status": "error", - "output": metadata, - "error": data.stderr, - } - }, - ) - return - - update_data( - output_dir, - {"exiftool": {"status": "ok", "output": metadata}}, - ) - except Exception as e: - update_data(output_dir, {"exiftool": {"status": "error", "error": str(e)}}) + analyzer = ExiftoolAnalyzer(input_img, output_dir) + analyzer.analyze() diff --git a/aperisolve/analyzers/foremost.py b/aperisolve/analyzers/foremost.py index eb0fc8c..45f96f6 100644 --- a/aperisolve/analyzers/foremost.py +++ b/aperisolve/analyzers/foremost.py @@ -3,60 +3,30 @@ # mypy: disable-error-code=unused-awaitable """Foremost Analyzer for Image Submissions.""" -import shutil -import subprocess from pathlib import Path -from .utils import MAX_PENDING_TIME, update_data +from .base_analyzer import SubprocessAnalyzer + + +class ForemostAnalyzer(SubprocessAnalyzer): + """Analyzer for foremost.""" + + def __init__(self, *args): + super().__init__("foremost", *args, has_archive=True) + self.cmd = ["foremost", "-o", str(self.get_extracted_dir()), "-i", self.img] + + def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: + """Check if the result is an error.""" + return len(stderr) > 60 + + def process_output(self, stdout: str, stderr: str) -> str | list[str] | dict[str, str]: + """Process the stdout/stderr.""" + if "Processing" in stderr and "|*|" in stderr: + return stderr.strip().split("\n") + return [] def analyze_foremost(input_img: Path, output_dir: Path) -> None: """Analyze an image submission using foremost.""" - - try: - foremost_dir = output_dir / "foremost" - foremost_dir.mkdir(parents=True, exist_ok=True) - - stderr = "" - # Run foremost - data = subprocess.run( - ["foremost", "-o", foremost_dir, "-i", input_img], - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - # Note: foremost use stderr as standard output for results :o) - # So we can't use it as error detection : - # stderr += data.stderr - - # Zip extracted files - zip_data = subprocess.run( - ["7z", "a", "../foremost.7z", "*"], - cwd=foremost_dir, - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - - stderr += zip_data.stderr - - # Remove the extracted directory - shutil.rmtree(foremost_dir) - - data_strings = data.stdout.split("\n") if data else [] - data_strings = [s for s in data_strings if s] - - update_data( - output_dir, - { - "foremost": { - "status": "ok", - "output": data_strings, - "download": f"/download/{output_dir.name}/foremost", - } - }, - ) - except Exception as e: - update_data(output_dir, {"foremost": {"status": "error", "error": str(e)}}) + analyzer = ForemostAnalyzer(input_img, output_dir) + analyzer.analyze() diff --git a/aperisolve/analyzers/outguess.py b/aperisolve/analyzers/outguess.py index 9fc6a78..738879e 100644 --- a/aperisolve/analyzers/outguess.py +++ b/aperisolve/analyzers/outguess.py @@ -3,97 +3,32 @@ # mypy: disable-error-code=unused-awaitable """Outguess Analyzer for Image Submissions.""" -import shutil -import subprocess from pathlib import Path from typing import Optional -from .utils import MAX_PENDING_TIME, update_data +from .base_analyzer import SubprocessAnalyzer -def analyze_outguess( - input_img: Path, output_dir: Path, password: Optional[str] = None -) -> None: - """Analyze an image submission using outguess.""" +class OutguessAnalyzer(SubprocessAnalyzer): + """Analyzer for outguess.""" - image_name = input_img.name - try: - outguess_dir = output_dir / "outguess" - outguess_dir.mkdir(parents=True, exist_ok=True) - outguess_file = "outguess.data" - stderr = "" - cmd: list[str] + def __init__(self, *args): + super().__init__("outguess", *args, has_archive=True) + + def build_cmd(self, password: Optional[str] = None) -> list[str]: + extracted_dir = self.get_extracted_dir() + assert extracted_dir is not None # since has_archive is True + out = str(extracted_dir / "outguess.data") if password: - cmd = [ - "outguess", - "-k", - password, - "-r", - "../../" + str(image_name), - outguess_file, - ] + return ["outguess", "-k", password, "-r", self.img, out] else: - cmd = ["outguess", "-r", "../../" + str(image_name), outguess_file] + return ["outguess", "-r", self.img, out] - data = subprocess.run( - cmd, - cwd=outguess_dir, - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - if data.returncode != 0: - stderr += data.stderr - err = { - "outguess": { - "status": "error", - "error": stderr, - } - } - update_data(output_dir, err) - return None - - # Zip extracted files - zip_data = subprocess.run( - ["7z", "a", "../outguess.7z", "*"], - cwd=outguess_dir, - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - - stderr += zip_data.stderr - - if len(stderr) > 0: - err = { - "outguess": { - "status": "error", - "error": stderr, - } - } - update_data(output_dir, err) - return None - - # Remove the extracted directory - shutil.rmtree(outguess_dir) - - data_strings = data.stderr.split("\n") if data else [] - data_strings = [s for s in data_strings if s] - - update_data( - output_dir, - { - "outguess": { - "status": "ok", - "output": data_strings, - "download": f"/download/{output_dir.name}/outguess", - } - }, - ) - - except Exception as e: - update_data(output_dir, {"outguess": {"status": "error", "error": str(e)}}) - return None +def analyze_outguess(input_img: Path, output_dir: Path, password: Optional[str] = None) -> None: + """Analyze an image submission using outguess.""" + analyzer = OutguessAnalyzer(input_img, output_dir) + if password: + analyzer.analyze(password) + else: + analyzer.analyze() diff --git a/aperisolve/analyzers/pngcheck.py b/aperisolve/analyzers/pngcheck.py index 730e67c..36ee677 100644 --- a/aperisolve/analyzers/pngcheck.py +++ b/aperisolve/analyzers/pngcheck.py @@ -3,54 +3,23 @@ # mypy: disable-error-code=unused-awaitable """Pngcheck Analyzer for Image Submissions.""" -import subprocess from pathlib import Path -from .utils import MAX_PENDING_TIME, update_data +from .base_analyzer import SubprocessAnalyzer + + +class PngcheckAnalyzer(SubprocessAnalyzer): + """Analyzer for pngcheck.""" + + def __init__(self, *args): + super().__init__("pngcheck", *args) + self.cmd = ["pngcheck", "-v", self.img] + + def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: + return "this is neither a PNG or JNG image nor a MNG stream" in stdout def analyze_pngcheck(input_img: Path, output_dir: Path) -> None: """Analyze an image submission using pngcheck.""" - - try: - data = subprocess.run( - ["pngcheck", "-v", input_img], - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - # Error based on https://github.com/pnggroup/pngcheck/issues/59 - # if data.stderr: - # update_data( - # output_dir, {"pngcheck": {"status": "error", "error": data.stderr}} - # ) - # return - - if "this is neither a PNG or JNG image nor a MNG stream" in data.stdout: - update_data( - output_dir, - { - "pngcheck": { - "status": "error", - "error": "ERRORS DETECTED: this is neither a PNG or JNG image nor a " - "MNG stream.", - } - }, - ) - return - - data_pngcheck = data.stdout.split("\n") if data else [] - data_pngcheck = [s for s in data_pngcheck if s] - - update_data( - output_dir, - { - "pngcheck": { - "status": "ok", - "output": data_pngcheck, - } - }, - ) - except Exception as e: - update_data(output_dir, {"pngcheck": {"status": "error", "error": str(e)}}) + analyzer = PngcheckAnalyzer(input_img, output_dir) + analyzer.analyze() diff --git a/aperisolve/analyzers/steghide.py b/aperisolve/analyzers/steghide.py index 0602bd6..0454e13 100644 --- a/aperisolve/analyzers/steghide.py +++ b/aperisolve/analyzers/steghide.py @@ -4,127 +4,67 @@ """Steghide Analyzer for Image Submissions.""" import re -import shutil -import subprocess from pathlib import Path from typing import Optional -from .utils import MAX_PENDING_TIME, update_data +from .base_analyzer import SubprocessAnalyzer -def analyze_steghide( - input_img: Path, output_dir: Path, password: Optional[str] = None -) -> None: - """Analyze an image submission using steghide.""" +class SteghideAnalyzer(SubprocessAnalyzer): + """Analyzer for steghide.""" - image_name = input_img.name - try: - stderr = "" - # Get embedded file information first - cmd: list[str] = ["steghide", "info", "../" + str(image_name)] - if password: - cmd += ["-p", password] - else: - cmd += ["-p", ""] + def __init__(self, *args): + super().__init__("steghide", *args, has_archive=True) - data = subprocess.run( - cmd, - cwd=output_dir, - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) + def build_cmd(self, password: Optional[str] = None) -> list[str]: + if password is None: + password = "" + + # First, get the embedded file name, and return the test cmd if failed + cmd_test = ["steghide", "info", self.img, "-p", password] + data = self.run_command(cmd_test, cwd=self.output_dir) + err = self.is_error(data.returncode, data.stdout, data.stderr, False) + if err: + return cmd_test match = re.search(r'embedded file "([^"]+)"', data.stdout) + assert match is not None + hidden = match.group(1) + cmd = ["steghide", "extract", "-sf", self.img, "-xf", hidden, "-p", password] + return cmd + + def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: + """Check if the result is an error.""" + + match = re.search(r'embedded file "([^"]+)"', stdout) embedded_filename = match.group(1) if match else None # if bad file format or wrong password, raise an error - if data.returncode != 0 or embedded_filename is None: - stderr += data.stderr.replace(f'"../{image_name}" ', "") # hide file name - # the file format of the file \"../foobar.png\" is not supported. - # become - # the file format of the file is not supported. - err = { - "steghide": { - "status": "error", - "error": stderr, - } - } - update_data(output_dir, err) - return None + return returncode != 0 or embedded_filename is None - # The passphrase is correct, we can extract the file - # prevent path traversal - embedded_file = output_dir / "steghide" / str(Path(embedded_filename).name) - extracted_dir = embedded_file.parent - extracted_dir.mkdir(parents=True, exist_ok=True) - - cmd_ext: list[str] = [ - "steghide", - "extract", - "-sf", - "../../" + str(image_name), - "-xf", - str(embedded_file.name), - ] - if password: - cmd_ext += ["-p", password] - else: - cmd_ext += ["-p", ""] - - data = subprocess.run( - cmd_ext, - cwd=extracted_dir, - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - # Filter out lines that indicate success - stdout = [] - for line in data.stderr.split("\n"): + def process_output(self, stdout: str, stderr: str) -> str | list[str] | dict[str, str]: + """Process the stdout into a list of lines.""" + out = [] + for line in stderr.split("\n"): if line.startswith('wrote extracted data to "'): - stdout.append(line) - elif line: - stderr += line + "\n" + out.append(line) + return out - # Zip extracted files - zip_data = subprocess.run( - ["7z", "a", "../steghide.7z", "*"], - cwd=extracted_dir, - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) + def process_error(self, stdout: str, stderr: str) -> str: + """Process stderr.""" + if "the file format of the file" in stderr and "not supported" in stderr: + return "The file format of the file is not supported (JPEG or BMP only)." + out = "" + for line in stderr.split("\n"): + if line and not line.startswith('wrote extracted data to "'): + out += line + "\n" + return out - stderr += zip_data.stderr - if len(stderr) > 0: - err = { - "steghide": { - "status": "error", - "error": stderr, - } - } - update_data(output_dir, err) - return None - - # Remove the extracted directory - shutil.rmtree(extracted_dir) - - update_data( - output_dir, - { - "steghide": { - "status": "ok", - "output": stdout, - "download": f"/download/{output_dir.name}/steghide", - } - }, - ) - - except Exception as e: - update_data(output_dir, {"steghide": {"status": "error", "error": str(e)}}) - return None +def analyze_steghide(input_img: Path, output_dir: Path, password: Optional[str] = None) -> None: + """Analyze an image submission using steghide.""" + analyzer = SteghideAnalyzer(input_img, output_dir) + if password: + analyzer.analyze(password) + else: + analyzer.analyze() diff --git a/aperisolve/analyzers/strings.py b/aperisolve/analyzers/strings.py index 6757fdb..0363814 100644 --- a/aperisolve/analyzers/strings.py +++ b/aperisolve/analyzers/strings.py @@ -3,45 +3,20 @@ # mypy: disable-error-code=unused-awaitable """Strings Analyzer for Image Submissions.""" -import subprocess from pathlib import Path -from .utils import MAX_PENDING_TIME, update_data +from .base_analyzer import SubprocessAnalyzer + + +class StringsAnalyzer(SubprocessAnalyzer): + """Analyzer for strings.""" + + def __init__(self, *args): + super().__init__("strings", *args) + self.cmd = ["strings", self.img] def analyze_strings(input_img: Path, output_dir: Path) -> None: """Analyze an image submission using strings.""" - - try: - data = subprocess.run( - ["strings", input_img], - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - if data.stderr: - update_data( - output_dir, {"strings": {"status": "error", "error": data.stderr}} - ) - return - - # strings_dir = output_dir / "strings" - # strings_dir.mkdir(parents=True, exist_ok=True) - # with open(strings_dir / "strings.txt", "w", encoding="utf-8") as f: - # subprocess.run(["strings", input_img], stdout=f, check=True) - - data_strings = data.stdout.split("\n") if data else [] - data_strings = [s for s in data_strings if s] - - update_data( - output_dir, - { - "strings": { - "status": "ok", - "output": data_strings, - } - }, - ) - except Exception as e: - update_data(output_dir, {"strings": {"status": "error", "error": str(e)}}) + analyzer = StringsAnalyzer(input_img, output_dir) + analyzer.analyze() diff --git a/aperisolve/analyzers/template_analyzer.py b/aperisolve/analyzers/template_analyzer.py index 0008497..8f33a34 100644 --- a/aperisolve/analyzers/template_analyzer.py +++ b/aperisolve/analyzers/template_analyzer.py @@ -1,79 +1,30 @@ # flake8: noqa: E203,E501,W503 # pylint: disable=C0413,W0718,R0903,R0801 # mypy: disable-error-code=unused-awaitable -"""Custom Analyzer for Image Submissions.""" +""" Analyzer for Image Submissions.""" -import shutil -import subprocess from pathlib import Path +from typing import Optional -from .utils import MAX_PENDING_TIME, update_data +from .base_analyzer import SubprocessAnalyzer -def analyze_myanalyzer(input_img: Path, output_dir: Path) -> None: - """Analyze an image submission using myanalyzer.""" +class TemplateAnalyze(SubprocessAnalyzer): + """Analyzer for .""" - image_name = input_img.name + def __init__(self, *args): + super().__init__("", *args, has_archive=True) - # Dir generated by the tool - extracted_dir = output_dir / "myanalyzer_output_dir" + def build_cmd(self, password: Optional[str] = None) -> list[str]: + if password: + return ["", "-p", password, "-i", self.img] + return ["", "-i", self.img] - try: - stderr, stdout = "", "" - # Run myanalyzer - data = subprocess.run( - ["myanalyzer", "-e", "../" + str(image_name)], - cwd=output_dir, - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - - stderr += data.stderr - stdout += data.stdout - - if len(stderr) > 0: # Return error if error has been raised - err = { - "myanalyzer": { - "status": "error", - "error": stderr, - } - } - update_data(output_dir, err) - return None - - zip_exist = False - if extracted_dir.exists(): # If result file(s) exist, zip them - subprocess.run( - ["7z", "a", "../myanalyzer.7z", "*"], - cwd=extracted_dir, - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - zip_exist = True - - if extracted_dir.exists(): # Remove the extracted directory - shutil.rmtree(extracted_dir) - - output = data.stdout.split("\n") if data else [] - output_data = { - "myanalyzer": { - "status": "ok", - "output": output, - } - } - - if zip_exist: - output_data["myanalyzer"][ - "download" - ] = f"/download/{output_dir.name}/myanalyzer" - - update_data(output_dir, output_data) - - except Exception as e: - update_data(output_dir, {"myanalyzer": {"status": "error", "error": str(e)}}) - return None +def analyze_template(input_img: Path, output_dir: Path, password: Optional[str] = None) -> None: + """Analyze an image submission using .""" + analyzer = TemplateAnalyze(input_img, output_dir) + if password: + analyzer.analyze(password) + else: + analyzer.analyze() diff --git a/aperisolve/analyzers/zsteg.py b/aperisolve/analyzers/zsteg.py index 0435841..3546efb 100644 --- a/aperisolve/analyzers/zsteg.py +++ b/aperisolve/analyzers/zsteg.py @@ -1,43 +1,31 @@ # flake8: noqa: E203,E501,W503 # pylint: disable=C0413,W0718,R0903,R0801 # mypy: disable-error-code=unused-awaitable -"""Zsteg Analyzer for Image Submissions.""" +"""Strings Analyzer for Image Submissions.""" -import subprocess from pathlib import Path -from .utils import MAX_PENDING_TIME, update_data +from .base_analyzer import SubprocessAnalyzer + + +class ZstegAnalyzer(SubprocessAnalyzer): + """Analyzer for zsteg.""" + + def __init__(self, *args, **kwargs): + super().__init__("zsteg", *args, **kwargs) + self.cmd = ["zsteg", self.img] + + def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: + return bool(stderr) or "PNG::NotSupported" in stdout[:100] + + def process_error(self, stdout: str, stderr: str) -> str: + """Process the stderr.""" + if "PNG::NotSupported" in stdout[:100]: + return "The file format of the file is not supported (PNG only)." + return stderr def analyze_zsteg(input_img: Path, output_dir: Path) -> None: """Analyze an image submission using zsteg.""" - - try: - data = subprocess.run( - ["zsteg", input_img], - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - if data.stderr or "PNG::NotSupported" in data.stdout[:100]: - error_msg = data.stderr or data.stdout - update_data(output_dir, {"zsteg": {"status": "error", "error": error_msg}}) - return - - # zsteg_dir = output_dir / "zsteg" - # zsteg_dir.mkdir(parents=True, exist_ok=True) - # with open(zsteg_dir / "zsteg.txt", "w", encoding="utf-8") as f: - # subprocess.run(["zsteg", input_img], stdout=f, check=True) - - update_data( - output_dir, - { - "zsteg": { - "status": "ok", - "output": data.stdout.split("\n") if data else [], - } - }, - ) - except Exception as e: - update_data(output_dir, {"zsteg": {"status": "error", "error": str(e)}}) + analyzer = ZstegAnalyzer(input_img, output_dir) + analyzer.analyze() From 65edf2074bbebc7277660caaf05537475e52bec7 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Thu, 1 Jan 2026 23:31:53 +0100 Subject: [PATCH 054/102] change textarea css --- aperisolve/static/css/aperisolve.css | 6 +++++- aperisolve/static/js/aperisolve.js | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/aperisolve/static/css/aperisolve.css b/aperisolve/static/css/aperisolve.css index 4279412..62d1901 100644 --- a/aperisolve/static/css/aperisolve.css +++ b/aperisolve/static/css/aperisolve.css @@ -379,6 +379,10 @@ td { height: 400px; } +.a-foremost textarea { + height: 100px; +} + /* ======== Footer ======== */ footer { @@ -420,4 +424,4 @@ footer a i { } #httperror h1, #httperror h2{ text-align: center; -} \ No newline at end of file +} diff --git a/aperisolve/static/js/aperisolve.js b/aperisolve/static/js/aperisolve.js index 0294ead..76947e2 100644 --- a/aperisolve/static/js/aperisolve.js +++ b/aperisolve/static/js/aperisolve.js @@ -264,6 +264,7 @@ function parseResult(result) { for (const line of result[tool]["output"]) { texarea_content += escapeHtml(`${line}\n`); } + texarea_content = texarea_content.trim(); texarea_content += ``; texarea_content += ``; texarea_content += `
`; From 24fd977613c93f7bab5e2fafc47e89af05eab6c8 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Thu, 1 Jan 2026 23:32:10 +0100 Subject: [PATCH 055/102] clean utils --- aperisolve/analyzers/utils.py | 51 ++--------------------------------- 1 file changed, 2 insertions(+), 49 deletions(-) diff --git a/aperisolve/analyzers/utils.py b/aperisolve/analyzers/utils.py index 3387106..a2ee0d0 100644 --- a/aperisolve/analyzers/utils.py +++ b/aperisolve/analyzers/utils.py @@ -3,56 +3,11 @@ # mypy: disable-error-code=unused-awaitable """Utility functions for analyzers modules.""" -import fcntl -import json import math -import os import struct -import threading import zlib from functools import cached_property -from pathlib import Path -from typing import Any, Iterable, Self - -_thread_lock = threading.Lock() - -MAX_PENDING_TIME = int(os.getenv("MAX_PENDING_TIME", "600")) # 10 minutes by default - - -def update_data( - output_dir: Path, new_data: dict[Any, Any], json_filename: str = "results.json" -) -> None: - """Thread-safe and process-safe JSON update using lock file and atomic replace.""" - json_file = Path(output_dir) / json_filename - lock_file = json_file.with_suffix(json_file.suffix + ".lock") - tmp_file = json_file.with_suffix(json_file.suffix + ".tmp") - - json_file.parent.mkdir(parents=True, exist_ok=True) - - with _thread_lock: # synchronizes across threads - with open(lock_file, "w", encoding="utf-8") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) # synchronizes across processes - - try: - # Read existing JSON - data: dict[Any, Any] = {} - if json_file.exists(): - try: - with open(json_file, "r", encoding="utf-8") as f: - data = json.load(f) - except json.JSONDecodeError: - data = {} - - # Update with new data - data.update(new_data) - - # Write safely to a temp file - with open(tmp_file, "w", encoding="utf-8") as f: - json.dump(data, f, sort_keys=False) - - os.replace(tmp_file, json_file) # ensures file write is atomic - finally: - fcntl.flock(lock, fcntl.LOCK_UN) +from typing import Iterable, Self def get_resolutions() -> list[tuple[int, int]]: @@ -291,9 +246,7 @@ class PNG: crc = struct.unpack(">I", data[offset : offset + 4])[0] # Unpack IHDR fields - width, height, depth, color, _, _, interlace = struct.unpack( - ">IIBBBBB", ihdr_data - ) + width, height, depth, color, _, _, interlace = struct.unpack(">IIBBBBB", ihdr_data) return cls( size=(width, height), From c5f7efafc13c4e47f09abe8c38a6ef05336f31e1 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 01:06:39 +0100 Subject: [PATCH 056/102] WIP --- aperisolve/analyzers/pngcheck.py | 5 +++++ aperisolve/analyzers/steghide.py | 15 +++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/aperisolve/analyzers/pngcheck.py b/aperisolve/analyzers/pngcheck.py index 36ee677..d62fca6 100644 --- a/aperisolve/analyzers/pngcheck.py +++ b/aperisolve/analyzers/pngcheck.py @@ -18,6 +18,11 @@ class PngcheckAnalyzer(SubprocessAnalyzer): def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: return "this is neither a PNG or JNG image nor a MNG stream" in stdout + def process_error(self, stdout: str, stderr: str) -> str: + """Process the stderr.""" + if "this is neither a PNG or JNG image nor a MNG stream" in stdout: + return "The file format of the file is not supported (PNG or JNG only)." + return stdout def analyze_pngcheck(input_img: Path, output_dir: Path) -> None: """Analyze an image submission using pngcheck.""" diff --git a/aperisolve/analyzers/steghide.py b/aperisolve/analyzers/steghide.py index 0454e13..2f3058a 100644 --- a/aperisolve/analyzers/steghide.py +++ b/aperisolve/analyzers/steghide.py @@ -36,16 +36,19 @@ class SteghideAnalyzer(SubprocessAnalyzer): def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: """Check if the result is an error.""" - match = re.search(r'embedded file "([^"]+)"', stdout) + match = re.search(r'embedded file "([^"]+)".*', stdout + stderr) embedded_filename = match.group(1) if match else None + match = re.search(r'wrote extracted data to "([^"]+)".*', stdout + stderr) + extracted_filename = match.group(1) if match else None + # if bad file format or wrong password, raise an error - return returncode != 0 or embedded_filename is None + return embedded_filename is None and extracted_filename is None def process_output(self, stdout: str, stderr: str) -> str | list[str] | dict[str, str]: """Process the stdout into a list of lines.""" out = [] - for line in stderr.split("\n"): + for line in (stdout + '\n' + stderr).split("\n"): if line.startswith('wrote extracted data to "'): out.append(line) return out @@ -54,11 +57,7 @@ class SteghideAnalyzer(SubprocessAnalyzer): """Process stderr.""" if "the file format of the file" in stderr and "not supported" in stderr: return "The file format of the file is not supported (JPEG or BMP only)." - out = "" - for line in stderr.split("\n"): - if line and not line.startswith('wrote extracted data to "'): - out += line + "\n" - return out + return stderr def analyze_steghide(input_img: Path, output_dir: Path, password: Optional[str] = None) -> None: From 18e8ba2cc794e889aabd07ea55e2e2b6f081ca69 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 04:53:57 +0100 Subject: [PATCH 057/102] steghide fixed --- aperisolve/analyzers/steghide.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/aperisolve/analyzers/steghide.py b/aperisolve/analyzers/steghide.py index 2f3058a..8bd26dd 100644 --- a/aperisolve/analyzers/steghide.py +++ b/aperisolve/analyzers/steghide.py @@ -29,8 +29,9 @@ class SteghideAnalyzer(SubprocessAnalyzer): match = re.search(r'embedded file "([^"]+)"', data.stdout) assert match is not None - hidden = match.group(1) - cmd = ["steghide", "extract", "-sf", self.img, "-xf", hidden, "-p", password] + hidden_file = match.group(1) + outfile = str(self.get_extracted_dir() / hidden_file) + cmd = ["steghide", "extract", "-sf", self.img, "-xf", outfile, "-p", password] return cmd def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: @@ -49,7 +50,7 @@ class SteghideAnalyzer(SubprocessAnalyzer): """Process the stdout into a list of lines.""" out = [] for line in (stdout + '\n' + stderr).split("\n"): - if line.startswith('wrote extracted data to "'): + if 'wrote extracted data to' in line: out.append(line) return out From 122817f4ab6218906f920cca81c4eb2ce4ee0239 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 17:07:52 +0100 Subject: [PATCH 058/102] openstego working --- aperisolve/analyzers/openstego.py | 154 +++++++++-------------------- aperisolve/static/js/aperisolve.js | 5 +- 2 files changed, 50 insertions(+), 109 deletions(-) diff --git a/aperisolve/analyzers/openstego.py b/aperisolve/analyzers/openstego.py index aa6e989..fd33a0d 100644 --- a/aperisolve/analyzers/openstego.py +++ b/aperisolve/analyzers/openstego.py @@ -3,115 +3,57 @@ # mypy: disable-error-code=unused-awaitable """OpenStego Analyzer for Image Submissions.""" -import shutil -import subprocess from pathlib import Path +from typing import Any, Optional -from .utils import MAX_PENDING_TIME, update_data +from .base_analyzer import SubprocessAnalyzer + + +class OpenStegoAnalyzer(SubprocessAnalyzer): + """Analyzer for openstego.""" + + def __init__(self, *args): + super().__init__("openstego", *args, has_archive=True) + self.algo = 0 + + def build_cmd(self, password: Optional[str] = None) -> list[str]: + """Iterator that return command for OpenStego for every algorithms""" + if password is None: + password = "" + + algorithms = ["AES128", "AES256"] + algo = algorithms[self.algo] + self.algo += 1 + cmd = ["openstego", "extract", "-a", "randomlsb", "--cryptalgo", algo, "-sf"] + cmd += [self.img, "-xd", str(self.get_extracted_dir()), "-p", password] + return cmd + + def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: + """Check if the result is an error.""" + return ( + len(list(self.get_extracted_dir().glob("*"))) == 0 and "Extracted file: " not in stderr + ) + + def analyze(self, password: Optional[str] = None) -> None: + """Run the subprocess command and handle results.""" + result: dict[str, Any] + try: + result = self.get_results(password) + if result["status"] == "error": # check second algorithm if failed + result = self.get_results(password) + self.update_result(result) + except Exception as e: + self.update_result({"status": "error", "error": str(e)}) + + def process_output(self, stdout: str, stderr: str) -> str | list[str] | dict[str, str]: + """Process the stdout/stderr.""" + return stderr def analyze_openstego(input_img: Path, output_dir: Path, password: str = "") -> None: """Analyze an image submission using openstego.""" - - image_name = input_img.name - try: - stderr = "" - # Try to extract the embedded file - extracted_dir = output_dir / "openstego" - extracted_dir.mkdir(parents=True, exist_ok=True) - - for algo in ["AES128", "AES256"]: - cmd: list[str] = [ - "openstego", - "extract", - "-a", - "randomlsb", - "--cryptalgo", - algo, - "-sf", - "../" + str(image_name), - "-xd", - str(extracted_dir), - "-p", - password, - ] - - data = subprocess.run( - cmd, - cwd=output_dir, - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - - # Check if extraction was successful - if data.returncode != 0: - stderr += data.stderr.replace( - f'"../{image_name}" ', "" - ) # hide file name - err = { - "openstego": { - "status": "error", - "error": stderr, - } - } - update_data(output_dir, err) - return None - - # Find the extracted file - extracted_files = list(extracted_dir.glob("*")) - if not extracted_files: - err = { - "openstego": { - "status": "error", - "error": "No file extracted, password may be incorrect.", - } - } - update_data(output_dir, err) - return None - - # Filter stdout for success messages - stdout = [] - for f in extracted_files: - stdout.append(f"Recovered file: {f.name}") - - # Zip extracted files - zip_data = subprocess.run( - ["7z", "a", "../openstego.7z", "*"], - cwd=extracted_dir, - capture_output=True, - text=True, - check=False, - timeout=MAX_PENDING_TIME, - ) - - stderr += zip_data.stderr - - if len(stderr) > 0: - err = { - "openstego": { - "status": "error", - "error": stderr, - } - } - update_data(output_dir, err) - return None - - # Remove the extracted directory - shutil.rmtree(extracted_dir) - - update_data( - output_dir, - { - "openstego": { - "status": "ok", - "output": stdout, - "download": f"/download/{output_dir.name}/openstego", - } - }, - ) - - except Exception as e: - update_data(output_dir, {"openstego": {"status": "error", "error": str(e)}}) - return None + analyzer = OpenStegoAnalyzer(input_img, output_dir) + if password: + analyzer.analyze(password) + else: + analyzer.analyze() diff --git a/aperisolve/static/js/aperisolve.js b/aperisolve/static/js/aperisolve.js index 76947e2..06ec2e8 100644 --- a/aperisolve/static/js/aperisolve.js +++ b/aperisolve/static/js/aperisolve.js @@ -254,9 +254,8 @@ function parseResult(result) { // Parse text output if (typeof result[tool]["output"] === "string") { - analyzer.innerHTML += `
${escapeHtml(
-        result[tool]["output"]
-      )}
`; + output = escapeHtml(result[tool]["output"]); + analyzer.innerHTML += ``; } else if (Array.isArray(result[tool]["output"])) { if (result[tool]["output"].length > 0) { var texarea_content = `
`; From e27fbe65116e157d3f0a5a561242057a742491ce Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 17:08:04 +0100 Subject: [PATCH 059/102] add black args line length --- .github/workflows/lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6294c1e..e0c4365 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -34,6 +34,7 @@ jobs: uses: wearerequired/lint-action@v2 with: black: true + black_args: "--line-length=100" flake8: true flake8_args: "--ignore=E203,E501,W503" pylint: true From 07d800d5830af2b10e53db2037ef5c65ab3b9174 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 17:09:22 +0100 Subject: [PATCH 060/102] split get_results and analysze --- aperisolve/analyzers/base_analyzer.py | 77 +++++++++++++++------------ 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/aperisolve/analyzers/base_analyzer.py b/aperisolve/analyzers/base_analyzer.py index 6d3748c..63b77e2 100644 --- a/aperisolve/analyzers/base_analyzer.py +++ b/aperisolve/analyzers/base_analyzer.py @@ -104,6 +104,47 @@ class SubprocessAnalyzer(ABC): raise NotImplementedError("cmd must be set or build_cmd overridden") return self.cmd + def get_results(self, password: Optional[str] = None) -> dict[str, Any]: + """Get results of command before returning.""" + result: dict[str, Any] + extracted_dir = None + if self.has_archive: + extracted_dir = self.get_extracted_dir() + if self.make_folder: + extracted_dir.mkdir(parents=True, exist_ok=True) + + cmd: list[str] + if password: + cmd = list(map(str, self.build_cmd(password))) + else: + cmd = list(map(str, self.build_cmd())) + data = self.run_command(cmd, cwd=self.output_dir) + returncode = data.returncode + stderr = data.stderr + stdout = data.stdout + + zip_exist = False + if extracted_dir and extracted_dir.exists() and any(extracted_dir.iterdir()): + self.generate_archive(self.output_dir, extracted_dir) # keep archive + zip_exist = True + + if self.is_error(returncode, stdout, stderr, zip_exist): + result = { + "status": "error", + } + result["error"] = self.process_error(stdout, stderr) + else: + result = { + "status": "ok", + "output": self.process_output(stdout, stderr), + } + note: Optional[str] = self.process_note(stdout, stderr) + if note: + result["note"] = note + if zip_exist: + result["download"] = f"/download/{self.output_dir.name}/{self.name}" + return result + @overload def analyze(self) -> None: ... @@ -114,41 +155,7 @@ class SubprocessAnalyzer(ABC): """Run the subprocess command and handle results.""" result: dict[str, Any] try: - extracted_dir = None - if self.has_archive: - extracted_dir = self.get_extracted_dir() - if self.make_folder: - extracted_dir.mkdir(parents=True, exist_ok=True) - - if password: - cmd = self.build_cmd(password) - else: - cmd = self.build_cmd() - data = self.run_command(cmd, cwd=self.output_dir) - returncode = data.returncode - stderr = data.stderr - stdout = data.stdout - - zip_exist = False - if extracted_dir and extracted_dir.exists() and any(extracted_dir.iterdir()): - self.generate_archive(self.output_dir, extracted_dir) # keep archive - zip_exist = True - - if self.is_error(returncode, stdout, stderr, zip_exist): - result = { - "status": "error", - } - result["error"] = self.process_error(stdout, stderr) - else: - result = { - "status": "ok", - "output": self.process_output(stdout, stderr), - } - note: Optional[str] = self.process_note(stdout, stderr) - if note: - result["note"] = note - if zip_exist: - result["download"] = f"/download/{self.output_dir.name}/{self.name}" + result = self.get_results(password) self.update_result(result) except Exception as e: self.update_result({"status": "error", "error": str(e)}) From 2ee0e97a0df3a207a93548cb0d7d736b57fa92a6 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 17:29:21 +0100 Subject: [PATCH 061/102] Fix openstego when no password --- aperisolve/analyzers/openstego.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/aperisolve/analyzers/openstego.py b/aperisolve/analyzers/openstego.py index fd33a0d..66e4316 100644 --- a/aperisolve/analyzers/openstego.py +++ b/aperisolve/analyzers/openstego.py @@ -49,6 +49,12 @@ class OpenStegoAnalyzer(SubprocessAnalyzer): """Process the stdout/stderr.""" return stderr + def process_error(self, stdout: str, stderr: str) -> str: + """Process the stderr.""" + if "OpenStego is a steganography application" in stderr: + return "OpenStego needs a password to work." + return stderr + def analyze_openstego(input_img: Path, output_dir: Path, password: str = "") -> None: """Analyze an image submission using openstego.""" From 40421f51bfd5bb6485240012845ed29e61f64d72 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 17:29:35 +0100 Subject: [PATCH 062/102] decomposer is now working --- aperisolve/analyzers/decomposer.py | 148 ++++++++++++++++------------- 1 file changed, 81 insertions(+), 67 deletions(-) diff --git a/aperisolve/analyzers/decomposer.py b/aperisolve/analyzers/decomposer.py index cff14d5..8cee810 100644 --- a/aperisolve/analyzers/decomposer.py +++ b/aperisolve/analyzers/decomposer.py @@ -4,78 +4,92 @@ """Bits Decomposer Analyzer for Image Submissions.""" from pathlib import Path +from typing import Any, Optional import numpy as np from PIL import Image -from .utils import update_data +from .base_analyzer import SubprocessAnalyzer + + +class DecomposerAnalyzer(SubprocessAnalyzer): + """Analyzer for bits decomposer.""" + + def __init__(self, *args): + super().__init__("decomposer", *args) + + def get_results(self, _: Optional[str] = None) -> dict[str, Any]: + """Analyze an image submission using bits decomposition.""" + try: + img = Image.open(self.input_img) + converted = False + + if img.mode == "P": + img = img.convert("RGB") + converted = True + + img_np = np.array(img) + + # Handle grayscale, RGB, RGBA, etc. + if len(img_np.shape) == 2: + channels = 1 + else: + channels = img_np.shape[2] + + if channels > 1: + channel_names = ["Red", "Green", "Blue", "Alpha"] + else: + channel_names = ["Grayscale"] + + image_json = {} + + # Add Superimposed RGB bit planes + if channels >= 3: + superimposed_json = [] + for bit in range(8): + bit_mask = 1 << bit + rgb_planes = [((img_np[..., c] & bit_mask) >> bit) * 255 for c in range(3)] + rgb_image = np.stack(rgb_planes, axis=-1).astype(np.uint8) + rgb_img = Image.fromarray(rgb_image, mode="RGB") + img_name = f"superimposed_bit_{bit}.png" + out_path = self.output_dir / img_name + dl_path = Path(self.output_dir.name) / img_name + superimposed_json.append("/image/" + str(dl_path)) + rgb_img.save(out_path) + + image_json["Superimposed"] = superimposed_json + + # Process individual channels + for c in range(channels): + channel_data = img_np[..., c] if channels > 1 else img_np + channel_label = channel_names[c] + channel_json = [] + for bit in range(8): + bit_mask = 1 << bit + bit_plane = ((channel_data & bit_mask) >> bit) * 255 + bit_img = Image.fromarray(bit_plane.astype(np.uint8), mode="L") + img_name = f"{channel_label}_bit_{bit}.png" + out_path = self.output_dir / img_name + dl_path = Path(self.output_dir.name) / img_name + channel_json.append("/image/" + str(dl_path)) + bit_img.save(out_path) + image_json[channel_label] = channel_json + + output = { + "status": "ok", + "images": image_json, + } + if converted: + output["note"] = ( + "Image contains a color palette and was converted to RGB for processing." + ) + + return output + except Exception as e: + return {"status": "error", "error": str(e)} def analyze_decomposer(input_img: Path, output_dir: Path) -> None: """Analyze an image submission using bits decomposition.""" - img = Image.open(input_img) - converted = False - - if img.mode == "P": - img = img.convert("RGB") - converted = True - - img_np = np.array(img) - - # Handle grayscale, RGB, RGBA, etc. - if len(img_np.shape) == 2: - channels = 1 - else: - channels = img_np.shape[2] - - if channels > 1: - channel_names = ["Red", "Green", "Blue", "Alpha"] - else: - channel_names = ["Grayscale"] - - image_json = {} - - # Add Superimposed RGB bit planes - if channels >= 3: - superimposed_json = [] - for bit in range(8): - bit_mask = 1 << bit - rgb_planes = [((img_np[..., c] & bit_mask) >> bit) * 255 for c in range(3)] - rgb_image = np.stack(rgb_planes, axis=-1).astype(np.uint8) - rgb_img = Image.fromarray(rgb_image, mode="RGB") - img_name = f"superimposed_bit_{bit}.png" - out_path = output_dir / img_name - dl_path = Path(output_dir.name) / img_name - superimposed_json.append("/image/" + str(dl_path)) - rgb_img.save(out_path) - - image_json["Superimposed"] = superimposed_json - - # Process individual channels - for c in range(channels): - channel_data = img_np[..., c] if channels > 1 else img_np - channel_label = channel_names[c] - channel_json = [] - for bit in range(8): - bit_mask = 1 << bit - bit_plane = ((channel_data & bit_mask) >> bit) * 255 - bit_img = Image.fromarray(bit_plane.astype(np.uint8), mode="L") - img_name = f"{channel_label}_bit_{bit}.png" - out_path = output_dir / img_name - dl_path = Path(output_dir.name) / img_name - channel_json.append("/image/" + str(dl_path)) - bit_img.save(out_path) - image_json[channel_label] = channel_json - - output = { - "decomposer": { - "status": "ok", - "images": image_json, - } - } - if converted: - output["decomposer"][ - "note" - ] = "Image contains a color palette and was converted to RGB for processing." - - update_data(output_dir, output) + analyzer = DecomposerAnalyzer(input_img, output_dir) + analyzer.analyze() From 36b8b1cca0907c3f8c5b5baea25977bde173773b Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 20:00:49 +0100 Subject: [PATCH 063/102] image resize ok --- aperisolve/analyzers/image_resize.py | 296 ++++++++++++--------------- aperisolve/workers.py | 10 +- 2 files changed, 139 insertions(+), 167 deletions(-) diff --git a/aperisolve/analyzers/image_resize.py b/aperisolve/analyzers/image_resize.py index 17db9b5..26a6c9b 100644 --- a/aperisolve/analyzers/image_resize.py +++ b/aperisolve/analyzers/image_resize.py @@ -8,191 +8,163 @@ from typing import Any, List from ..app import app from ..models import IHDR -from .utils import PNG, update_data +from .base_analyzer import SubprocessAnalyzer +from .utils import PNG -def write_recovered_png( - png_bytes: bytearray, - width: int, - height: int, - output_path: Path, -) -> None: - """ - Rebuilds a PNG by patching IHDR width/height and writes it to disk. - """ - height_bytes = height.to_bytes(4, byteorder="big") - width_bytes = width.to_bytes(4, byteorder="big") +class ResizeAnalyzer(SubprocessAnalyzer): + """Analyzer for image resize.""" - full_png_data = ( - png_bytes[:16] # Splicing: [Start...IHDR+4] + [W] + [H] + [IHDR+12...End] - + width_bytes - + height_bytes - + png_bytes[24:] - ) + png: PNG - with output_path.open("wb") as out_f: - out_f.write(full_png_data) + def __init__(self, *args): + super().__init__("image_resize", *args, has_archive=True) + def _lookup_crc(self) -> List[PNG]: + """ + Look up PNG images in the database by their CRC value. -def lookup_crc(png: PNG) -> List[PNG]: - """ - Look up PNG images in the database by their CRC value. + Returns: + List[PNG]: A list of PNG objects with matching CRC values, reconstructed from their + packed representations. + """ + with app.app_context(): + imgs = IHDR.query.filter_by(crc=self.png.crc).all() + results = [] + for img in imgs: + packed = img.packed + results.append(PNG.from_packed(packed, crc=self.png.crc)) + return results - Args: - png (PNG): A PNG object containing the CRC value to search for. + def _search_height_crc(self) -> List[PNG]: + """ + Search for PNG images with matching CRC by iterating through possible heights. - Returns: - List[PNG]: A list of PNG objects with matching CRC values, reconstructed from their packed - representations. - """ - with app.app_context(): - imgs = IHDR.query.filter_by(crc=png.crc).all() + This function attempts to find alternative PNG configurations that produce the same + CRC checksum as the input PNG by systematically varying the height parameter while + keeping all other properties constant. + + Returns: + List[PNG]: A list of PNG objects with matching CRC values. Returns an empty list + if no matches are found. + + Note: + The function iterates through heights from 100 to 3500 (inclusive). + This may be computationally expensive for large ranges. + """ results = [] - for img in imgs: - packed = img.packed - results.append(PNG.from_packed(packed, crc=png.crc)) - return results - - -def search_height_crc(png: PNG) -> List[PNG]: - """ - Search for PNG images with matching CRC by iterating through possible heights. - - This function attempts to find alternative PNG configurations that produce the same - CRC checksum as the input PNG by systematically varying the height parameter while - keeping all other properties constant. - - Args: - png (PNG): The reference PNG object whose CRC checksum will be matched. - - Returns: - List[PNG]: A list of PNG objects with matching CRC values. Returns an empty list - if no matches are found. - - Note: - The function iterates through heights from 100 to 3500 (inclusive). - This may be computationally expensive for large ranges. - """ - results = [] - for height in range(100, 3501): - candidate = PNG( - size=(png.width, height), - bit_depth=png.bit_depth, - color_type=png.color_type, - interlace=png.interlace, - ) - if candidate.crc == png.crc: - results.append(candidate) - return results - - -def analyze_image_resize(input_img: Path, output_dir: Path) -> None: - """ - Analyze a PNG image and recover resized versions based on CRC matching. - - This function attempts to find matching image dimensions for a given PNG file - by looking up its CRC (Cyclic Redundancy Check) in a database. If matches are - found, it recovers and saves PNG images with the matched dimensions. - - Args: - input_img (Path): Path to the input PNG image file to analyze. - output_dir (Path): Path to the directory where recovered PNG images will be saved. - - Returns: - None - - Raises: - Handles exceptions gracefully by updating output data with error status. - - Side Effects: - - Creates output directory if it doesn't exist. - - Saves recovered PNG images to the output directory. - - Updates data in the output directory with analysis results via update_data(). - - Notes: - - If the PNG has invalid structure or IHDR chunk is not first, an error is logged. - - The function performs a two-step lookup: first by CRC, then by height/CRC if no matches - found. - - Recovered images are named as "recovered_{width}x{height}.png". - - Analysis results (logs, status, and image URLs) are persisted via update_data(). - """ - - # 1. Setup Paths - input_img = Path(input_img) - output_dir = Path(output_dir) - - # 3. Initialize Logs - logs = [] - - try: - with input_img.open("rb") as image: - img_bytes = bytearray(image.read()) - try: - png = PNG.from_bytearray(img_bytes) - except ValueError: - png = None - - if png is None: - logs.append( - "Failure: IHDR chunk is not the first chunk, or PNG has invalid structure." + for height in range(100, 3501): + candidate = PNG( + size=(self.png.width, height), + bit_depth=self.png.bit_depth, + color_type=self.png.color_type, + interlace=self.png.interlace, ) - update_data( - output_dir, - { - "image_resize": { + if candidate.crc == self.png.crc: + results.append(candidate) + return results + + def _write_recovered_png(self, png_bytes: bytearray, png_match: PNG) -> str: + """ + Rebuilds a PNG by patching IHDR width/height and writes it to disk. + """ + + img_name = f"recovered_{png_match.width}x{png_match.height}.png" + output_path = self.output_dir / img_name + + height_bytes = png_match.height.to_bytes(4, byteorder="big") + width_bytes = png_match.width.to_bytes(4, byteorder="big") + + full_png_data = ( + png_bytes[:16] # Splicing: [Start...IHDR+4] + [W] + [H] + [IHDR+12...End] + + width_bytes + + height_bytes + + png_bytes[24:] + ) + + with output_path.open("wb") as out_f: + out_f.write(full_png_data) + + return img_name + + def get_results(self, _: Optional[str] = None) -> dict[str, Any]: + """ + Analyze a PNG image and recover resized versions based on CRC matching. + + This function attempts to find matching image dimensions for a given PNG file + by looking up its CRC (Cyclic Redundancy Check) in a database. If matches are + found, it recovers and saves PNG images with the matched dimensions. + + Returns: + None + + Raises: + Handles exceptions gracefully by updating output data with error status. + + Side Effects: + - Creates output directory if it doesn't exist. + - Saves recovered PNG images to the output directory. + - Updates data in the output directory with analysis results via update_data(). + + Notes: + - If the PNG has invalid structure or IHDR chunk is not first, an error is logged. + - The function performs a two-step lookup: first by CRC, then by height/CRC if no + matches found. + - Recovered images are named as "recovered_{width}x{height}.png". + - Analysis results (logs, status, and image URLs) are persisted via update_data(). + """ + logs = [] + + try: + with self.input_img.open("rb") as image: + img_bytes = bytearray(image.read()) + try: + self.png = PNG.from_bytearray(img_bytes) + except ValueError: + return { "status": "error", "error": "IHDR chunk is not the first chunk, or PNG has invalid structure.", } - }, - ) - return - logs.append(f"Target CRC found: 0x{png.crc:08x}") + # Already correct + if self.png.crc == self.png.compute_crc(): + return { + "status": "ok", + "note": f"PNG is already valid with dimensions " + f"{self.png.width}x{self.png.height} and crc 0x{self.png.crc:08x}.", + } - if png.crc == png.compute_crc(): - note = ( - f"PNG is already valid with dimensions {png.width}x{png.height} " - f"and crc 0x{png.crc:08x}." - ) - output_no_compute: dict[str, Any] = { - "image_resize": {"status": "ok", "note": note} - } - update_data(output_dir, output_no_compute) - return + saved_img_urls = [] - saved_img_urls = [] - - db_matches = lookup_crc(png) - if not db_matches: - db_matches = search_height_crc(png) - if not db_matches: - logs.append("Failure: No matching dimensions found.") - else: - # Create parent directories if they don't exist - output_dir.mkdir(parents=True, exist_ok=True) - - # Iterate through ALL candidates found - for im_match in db_matches: - img_name = f"recovered_{im_match.width}x{im_match.height}.png" - output_path = output_dir / img_name - write_recovered_png( - img_bytes, im_match.width, im_match.height, output_path - ) + # No match found + if not (db_matches := self._lookup_crc()) and not ( + db_matches := self._search_height_crc() + ): + return { + "status": "error", + "error": "Failure: No matching dimensions found.", + } + # Match found, saving image(s) + logs.append(f"Target CRC found: 0x{self.png.crc:08x}") + self.output_dir.mkdir(parents=True, exist_ok=True) + for png_match in db_matches: + img_name = self._write_recovered_png(img_bytes, png_match) logs.append(f"Image saved: {img_name}") # Append the formatted URL to our list - saved_img_urls.append("/image/" + str(Path(output_dir.name) / img_name)) + saved_img_urls.append("/image/" + str(Path(self.output_dir.name) / img_name)) - # 4. Final Data Update - output_data: dict[str, Any] = { - "image_resize": { + return { "status": "ok", "output": logs, "png_images": saved_img_urls, } - } - update_data(output_dir, output_data) + except Exception as e: + return {"status": "error", "error": str(e)} - except Exception as e: - update_data(output_dir, {"image_resize": {"status": "error", "error": str(e)}}) + +def analyze_image_resize(input_img: Path, output_dir: Path) -> None: + """Analyze an image submission using outguess.""" + analyzer = ResizeAnalyzer(input_img, output_dir) + analyzer.analyze() diff --git a/aperisolve/workers.py b/aperisolve/workers.py index 72b7861..8ab66ef 100644 --- a/aperisolve/workers.py +++ b/aperisolve/workers.py @@ -89,12 +89,12 @@ def analyze_image(submission_hash: str) -> None: (analyze_decomposer, img_path, result_path), (analyze_exiftool, img_path, result_path), (analyze_foremost, img_path, result_path), - (analyze_strings, img_path, result_path), - (analyze_pngcheck, img_path, result_path), - (analyze_steghide, img_path, result_path, submission.password), - (analyze_openstego, img_path, result_path, submission.password), - (analyze_zsteg, img_path, result_path), (analyze_image_resize, img_path, result_path), + (analyze_openstego, img_path, result_path, submission.password), + (analyze_pngcheck, img_path, result_path), + (analyze_strings, img_path, result_path), + (analyze_steghide, img_path, result_path, submission.password), + (analyze_zsteg, img_path, result_path), ] # Deep analysis only From 12402a381f50572080afb05cec45c56a1c8662bc Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 20:04:06 +0100 Subject: [PATCH 064/102] Black --- aperisolve/analyzers/pngcheck.py | 1 + aperisolve/analyzers/steghide.py | 4 ++-- aperisolve/app.py | 21 +++++---------------- aperisolve/cleanup.py | 4 +--- aperisolve/models.py | 4 +--- aperisolve/sentry.py | 4 +--- 6 files changed, 11 insertions(+), 27 deletions(-) diff --git a/aperisolve/analyzers/pngcheck.py b/aperisolve/analyzers/pngcheck.py index d62fca6..6e04559 100644 --- a/aperisolve/analyzers/pngcheck.py +++ b/aperisolve/analyzers/pngcheck.py @@ -24,6 +24,7 @@ class PngcheckAnalyzer(SubprocessAnalyzer): return "The file format of the file is not supported (PNG or JNG only)." return stdout + def analyze_pngcheck(input_img: Path, output_dir: Path) -> None: """Analyze an image submission using pngcheck.""" analyzer = PngcheckAnalyzer(input_img, output_dir) diff --git a/aperisolve/analyzers/steghide.py b/aperisolve/analyzers/steghide.py index 8bd26dd..fec8f5d 100644 --- a/aperisolve/analyzers/steghide.py +++ b/aperisolve/analyzers/steghide.py @@ -49,8 +49,8 @@ class SteghideAnalyzer(SubprocessAnalyzer): def process_output(self, stdout: str, stderr: str) -> str | list[str] | dict[str, str]: """Process the stdout into a list of lines.""" out = [] - for line in (stdout + '\n' + stderr).split("\n"): - if 'wrote extracted data to' in line: + for line in (stdout + "\n" + stderr).split("\n"): + if "wrote extracted data to" in line: out.append(line) return out diff --git a/aperisolve/app.py b/aperisolve/app.py index 734097c..ae869ea 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -28,9 +28,7 @@ app: Flask = Flask(__name__) app.json.sort_keys = False # type: ignore app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get("DB_URI") app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False -app.config["MAX_CONTENT_LENGTH"] = int( - os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024) -) +app.config["MAX_CONTENT_LENGTH"] = int(os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024)) RESULT_FOLDER.mkdir(parents=True, exist_ok=True) db.init_app(app) @@ -112,10 +110,7 @@ def upload_image() -> tuple[Response, int]: if image.filename is None or image.filename == "": return jsonify({"error": "No image provided"}), 400 - if ( - "." not in image.filename - or Path(image.filename).suffix.lower() not in IMAGE_EXTENSIONS - ): + if "." not in image.filename or Path(image.filename).suffix.lower() not in IMAGE_EXTENSIONS: return jsonify({"error": "Unsupported file type"}), 400 # Compute hashes and prepare submission @@ -230,9 +225,7 @@ def get_result(hash_val: str) -> tuple[Response, int]: submission = Submission.query.get_or_404(hash_val) image = Image.query.get_or_404(submission.image_hash) # type: ignore - result_path = ( - RESULT_FOLDER / str(image.hash) / str(submission.hash) / "results.json" - ) + result_path = RESULT_FOLDER / str(image.hash) / str(submission.hash) / "results.json" if not result_path.exists(): return jsonify({"error": "Results not ready yet..."}), 425 @@ -261,9 +254,7 @@ def download_output(hash_val: str, tool: str) -> Response: @app.route("/image/") @app.route("/image//") -def get_image( - hash_val: Optional[str] = None, img_name: Optional[str] = None -) -> Response: +def get_image(hash_val: Optional[str] = None, img_name: Optional[str] = None) -> Response: """ Download the image with a specific name (usually ones generated with decomposer) for a given submission hash. @@ -283,9 +274,7 @@ def get_image( image = Image.query.filter_by(hash=hash_val).first_or_404() output_file = Path(image.file) # type: ignore - if (not output_file.exists()) or ( - output_file.suffix.lower() not in IMAGE_EXTENSIONS - ): + if (not output_file.exists()) or (output_file.suffix.lower() not in IMAGE_EXTENSIONS): return abort(404, description="Image not found or unsupported format") return send_file(output_file, as_attachment=True) diff --git a/aperisolve/cleanup.py b/aperisolve/cleanup.py index 5a63f96..032a7b6 100644 --- a/aperisolve/cleanup.py +++ b/aperisolve/cleanup.py @@ -49,9 +49,7 @@ def cleanup_old_entries() -> None: db.session.delete(submission) # pylint: disable=no-member elif submission.status == "done": # type: ignore # Search for buggy results, delete - result_path = ( - RESULT_FOLDER / str(submission.image_hash) / str(submission.hash) - ) + result_path = RESULT_FOLDER / str(submission.image_hash) / str(submission.hash) result_file = result_path / "results.json" if not result_file.exists(): db.session.delete(submission) # pylint: disable=no-member diff --git a/aperisolve/models.py b/aperisolve/models.py index dc47053..22d6d34 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -97,9 +97,7 @@ def fill_ihdr_db() -> None: if exists: # Skip if already present continue - db.session.add( # pylint: disable=no-member - IHDR(packed=p.packed, crc=p.crc) - ) + db.session.add(IHDR(packed=p.packed, crc=p.crc)) # pylint: disable=no-member count += 1 if count % 5000 == 0: diff --git a/aperisolve/sentry.py b/aperisolve/sentry.py index 13b5871..c0307a5 100644 --- a/aperisolve/sentry.py +++ b/aperisolve/sentry.py @@ -14,9 +14,7 @@ from sentry_sdk.integrations.threading import ThreadingIntegration SENTRY_DSN = os.environ.get("SENTRY_DSN") SENTRY_ENVIRONMENT = os.environ.get("SENTRY_ENVIRONMENT", "development") SENTRY_TRACES_SAMPLE_RATE = float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0.1")) -SENTRY_PROFILES_SAMPLE_RATE = float( - os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1") -) +SENTRY_PROFILES_SAMPLE_RATE = float(os.environ.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1")) SENTRY_RELEASE = os.environ.get("SENTRY_RELEASE", "1.0.0") From e5c0ac55efc8ccac9638dbee0174eccb718520d7 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 20:43:45 +0100 Subject: [PATCH 065/102] pylint --- aperisolve/analyzers/image_resize.py | 2 +- aperisolve/analyzers/outguess.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/aperisolve/analyzers/image_resize.py b/aperisolve/analyzers/image_resize.py index 26a6c9b..99d8077 100644 --- a/aperisolve/analyzers/image_resize.py +++ b/aperisolve/analyzers/image_resize.py @@ -4,7 +4,7 @@ """Image size bruteforcer for common PNG sizes.""" from pathlib import Path -from typing import Any, List +from typing import Any, List, Optional from ..app import app from ..models import IHDR diff --git a/aperisolve/analyzers/outguess.py b/aperisolve/analyzers/outguess.py index 738879e..2557eb0 100644 --- a/aperisolve/analyzers/outguess.py +++ b/aperisolve/analyzers/outguess.py @@ -21,8 +21,7 @@ class OutguessAnalyzer(SubprocessAnalyzer): out = str(extracted_dir / "outguess.data") if password: return ["outguess", "-k", password, "-r", self.img, out] - else: - return ["outguess", "-r", self.img, out] + return ["outguess", "-r", self.img, out] def analyze_outguess(input_img: Path, output_dir: Path, password: Optional[str] = None) -> None: From 2e602959ce221d627e705b891fdfec06a7eed47f Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 20:51:44 +0100 Subject: [PATCH 066/102] mypy --- aperisolve/analyzers/base_analyzer.py | 4 +++- aperisolve/analyzers/binwalk.py | 2 +- aperisolve/analyzers/decomposer.py | 2 +- aperisolve/analyzers/exiftool.py | 3 ++- aperisolve/analyzers/foremost.py | 3 ++- aperisolve/analyzers/image_resize.py | 2 +- aperisolve/analyzers/openstego.py | 2 +- aperisolve/analyzers/outguess.py | 4 ++-- aperisolve/analyzers/pngcheck.py | 3 ++- aperisolve/analyzers/steghide.py | 4 ++-- aperisolve/analyzers/strings.py | 3 ++- aperisolve/analyzers/template_analyzer.py | 4 ++-- aperisolve/analyzers/zsteg.py | 5 +++-- 13 files changed, 24 insertions(+), 17 deletions(-) diff --git a/aperisolve/analyzers/base_analyzer.py b/aperisolve/analyzers/base_analyzer.py index 63b77e2..5f20a05 100644 --- a/aperisolve/analyzers/base_analyzer.py +++ b/aperisolve/analyzers/base_analyzer.py @@ -28,7 +28,9 @@ class SubprocessAnalyzer(ABC): img: str make_folder: bool = True - def __init__(self, name: str, input_img: Path, output_dir: Path, has_archive: bool = False): + def __init__( + self, name: str, input_img: Path, output_dir: Path, has_archive: bool = False + ) -> None: self.name = name self.input_img = input_img self.img = f"../{self.input_img.name}" diff --git a/aperisolve/analyzers/binwalk.py b/aperisolve/analyzers/binwalk.py index eb04845..f26d42a 100644 --- a/aperisolve/analyzers/binwalk.py +++ b/aperisolve/analyzers/binwalk.py @@ -11,7 +11,7 @@ from .base_analyzer import SubprocessAnalyzer class BinwalkAnalyzer(SubprocessAnalyzer): """Analyzer for binwalk.""" - def __init__(self, *args): + def __init__(self, *args) -> None: super().__init__("binwalk", *args, has_archive=True) self.cmd = ["binwalk", "--matryoshka", "-e", self.img, "--run-as=root"] self.make_folder = False # Binwalk already create its own folder diff --git a/aperisolve/analyzers/decomposer.py b/aperisolve/analyzers/decomposer.py index 8cee810..aba4fce 100644 --- a/aperisolve/analyzers/decomposer.py +++ b/aperisolve/analyzers/decomposer.py @@ -15,7 +15,7 @@ from .base_analyzer import SubprocessAnalyzer class DecomposerAnalyzer(SubprocessAnalyzer): """Analyzer for bits decomposer.""" - def __init__(self, *args): + def __init__(self, *args: Any) -> None: super().__init__("decomposer", *args) def get_results(self, _: Optional[str] = None) -> dict[str, Any]: diff --git a/aperisolve/analyzers/exiftool.py b/aperisolve/analyzers/exiftool.py index ae42bae..b749a8d 100644 --- a/aperisolve/analyzers/exiftool.py +++ b/aperisolve/analyzers/exiftool.py @@ -4,6 +4,7 @@ """Exiftool Analyzer for Image Submissions.""" from pathlib import Path +from typing import Any from .base_analyzer import SubprocessAnalyzer @@ -11,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class ExiftoolAnalyzer(SubprocessAnalyzer): """Analyzer for exiftool.""" - def __init__(self, *args): + def __init__(self, *args: Any) -> None: super().__init__("exiftool", *args) self.cmd = ["exiftool", "-a", "-u", "-g1", self.img] diff --git a/aperisolve/analyzers/foremost.py b/aperisolve/analyzers/foremost.py index 45f96f6..8900c28 100644 --- a/aperisolve/analyzers/foremost.py +++ b/aperisolve/analyzers/foremost.py @@ -4,6 +4,7 @@ """Foremost Analyzer for Image Submissions.""" from pathlib import Path +from typing import Any from .base_analyzer import SubprocessAnalyzer @@ -11,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class ForemostAnalyzer(SubprocessAnalyzer): """Analyzer for foremost.""" - def __init__(self, *args): + def __init__(self, *args: Any) -> None: super().__init__("foremost", *args, has_archive=True) self.cmd = ["foremost", "-o", str(self.get_extracted_dir()), "-i", self.img] diff --git a/aperisolve/analyzers/image_resize.py b/aperisolve/analyzers/image_resize.py index 99d8077..f9241b2 100644 --- a/aperisolve/analyzers/image_resize.py +++ b/aperisolve/analyzers/image_resize.py @@ -17,7 +17,7 @@ class ResizeAnalyzer(SubprocessAnalyzer): png: PNG - def __init__(self, *args): + def __init__(self, *args: Any) -> None: super().__init__("image_resize", *args, has_archive=True) def _lookup_crc(self) -> List[PNG]: diff --git a/aperisolve/analyzers/openstego.py b/aperisolve/analyzers/openstego.py index 66e4316..26f6960 100644 --- a/aperisolve/analyzers/openstego.py +++ b/aperisolve/analyzers/openstego.py @@ -12,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class OpenStegoAnalyzer(SubprocessAnalyzer): """Analyzer for openstego.""" - def __init__(self, *args): + def __init__(self, *args: Any) -> None: super().__init__("openstego", *args, has_archive=True) self.algo = 0 diff --git a/aperisolve/analyzers/outguess.py b/aperisolve/analyzers/outguess.py index 2557eb0..cb8ec9d 100644 --- a/aperisolve/analyzers/outguess.py +++ b/aperisolve/analyzers/outguess.py @@ -4,7 +4,7 @@ """Outguess Analyzer for Image Submissions.""" from pathlib import Path -from typing import Optional +from typing import Any, Optional from .base_analyzer import SubprocessAnalyzer @@ -12,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class OutguessAnalyzer(SubprocessAnalyzer): """Analyzer for outguess.""" - def __init__(self, *args): + def __init__(self, *args: Any) -> None: super().__init__("outguess", *args, has_archive=True) def build_cmd(self, password: Optional[str] = None) -> list[str]: diff --git a/aperisolve/analyzers/pngcheck.py b/aperisolve/analyzers/pngcheck.py index 6e04559..03350e1 100644 --- a/aperisolve/analyzers/pngcheck.py +++ b/aperisolve/analyzers/pngcheck.py @@ -4,6 +4,7 @@ """Pngcheck Analyzer for Image Submissions.""" from pathlib import Path +from typing import Any from .base_analyzer import SubprocessAnalyzer @@ -11,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class PngcheckAnalyzer(SubprocessAnalyzer): """Analyzer for pngcheck.""" - def __init__(self, *args): + def __init__(self, *args: Any) -> None: super().__init__("pngcheck", *args) self.cmd = ["pngcheck", "-v", self.img] diff --git a/aperisolve/analyzers/steghide.py b/aperisolve/analyzers/steghide.py index fec8f5d..fb160fe 100644 --- a/aperisolve/analyzers/steghide.py +++ b/aperisolve/analyzers/steghide.py @@ -5,7 +5,7 @@ import re from pathlib import Path -from typing import Optional +from typing import Any, Optional from .base_analyzer import SubprocessAnalyzer @@ -13,7 +13,7 @@ from .base_analyzer import SubprocessAnalyzer class SteghideAnalyzer(SubprocessAnalyzer): """Analyzer for steghide.""" - def __init__(self, *args): + def __init__(self, *args: Any) -> None: super().__init__("steghide", *args, has_archive=True) def build_cmd(self, password: Optional[str] = None) -> list[str]: diff --git a/aperisolve/analyzers/strings.py b/aperisolve/analyzers/strings.py index 0363814..2285bbf 100644 --- a/aperisolve/analyzers/strings.py +++ b/aperisolve/analyzers/strings.py @@ -4,6 +4,7 @@ """Strings Analyzer for Image Submissions.""" from pathlib import Path +from typing import Any from .base_analyzer import SubprocessAnalyzer @@ -11,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class StringsAnalyzer(SubprocessAnalyzer): """Analyzer for strings.""" - def __init__(self, *args): + def __init__(self, *args: Any) -> None: super().__init__("strings", *args) self.cmd = ["strings", self.img] diff --git a/aperisolve/analyzers/template_analyzer.py b/aperisolve/analyzers/template_analyzer.py index 8f33a34..70a743f 100644 --- a/aperisolve/analyzers/template_analyzer.py +++ b/aperisolve/analyzers/template_analyzer.py @@ -4,7 +4,7 @@ """ Analyzer for Image Submissions.""" from pathlib import Path -from typing import Optional +from typing import Any, Optional from .base_analyzer import SubprocessAnalyzer @@ -12,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class TemplateAnalyze(SubprocessAnalyzer): """Analyzer for .""" - def __init__(self, *args): + def __init__(self, *args: Any) -> None: super().__init__("", *args, has_archive=True) def build_cmd(self, password: Optional[str] = None) -> list[str]: diff --git a/aperisolve/analyzers/zsteg.py b/aperisolve/analyzers/zsteg.py index 3546efb..d7de261 100644 --- a/aperisolve/analyzers/zsteg.py +++ b/aperisolve/analyzers/zsteg.py @@ -4,6 +4,7 @@ """Strings Analyzer for Image Submissions.""" from pathlib import Path +from typing import Any from .base_analyzer import SubprocessAnalyzer @@ -11,8 +12,8 @@ from .base_analyzer import SubprocessAnalyzer class ZstegAnalyzer(SubprocessAnalyzer): """Analyzer for zsteg.""" - def __init__(self, *args, **kwargs): - super().__init__("zsteg", *args, **kwargs) + def __init__(self, *args: Any) -> None: + super().__init__("zsteg", *args) self.cmd = ["zsteg", self.img] def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: From 456e041892234ec52bcfa25fc7af7b6eb898889f Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 21:01:44 +0100 Subject: [PATCH 067/102] fix mypy --- aperisolve/analyzers/binwalk.py | 4 ++-- aperisolve/analyzers/decomposer.py | 4 ++-- aperisolve/analyzers/exiftool.py | 4 ++-- aperisolve/analyzers/foremost.py | 4 ++-- aperisolve/analyzers/image_resize.py | 4 ++-- aperisolve/analyzers/openstego.py | 4 ++-- aperisolve/analyzers/outguess.py | 4 ++-- aperisolve/analyzers/pngcheck.py | 4 ++-- aperisolve/analyzers/steghide.py | 4 ++-- aperisolve/analyzers/strings.py | 4 ++-- aperisolve/analyzers/template_analyzer.py | 4 ++-- aperisolve/analyzers/zsteg.py | 4 ++-- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/aperisolve/analyzers/binwalk.py b/aperisolve/analyzers/binwalk.py index f26d42a..7b4e8c0 100644 --- a/aperisolve/analyzers/binwalk.py +++ b/aperisolve/analyzers/binwalk.py @@ -11,8 +11,8 @@ from .base_analyzer import SubprocessAnalyzer class BinwalkAnalyzer(SubprocessAnalyzer): """Analyzer for binwalk.""" - def __init__(self, *args) -> None: - super().__init__("binwalk", *args, has_archive=True) + def __init__(self, input_img, output_dir) -> None: + super().__init__("binwalk", input_img, output_dir, has_archive=True) self.cmd = ["binwalk", "--matryoshka", "-e", self.img, "--run-as=root"] self.make_folder = False # Binwalk already create its own folder diff --git a/aperisolve/analyzers/decomposer.py b/aperisolve/analyzers/decomposer.py index aba4fce..a9de9c6 100644 --- a/aperisolve/analyzers/decomposer.py +++ b/aperisolve/analyzers/decomposer.py @@ -15,8 +15,8 @@ from .base_analyzer import SubprocessAnalyzer class DecomposerAnalyzer(SubprocessAnalyzer): """Analyzer for bits decomposer.""" - def __init__(self, *args: Any) -> None: - super().__init__("decomposer", *args) + def __init__(self, input_img, output_dir: Any) -> None: + super().__init__("decomposer", input_img, output_dir) def get_results(self, _: Optional[str] = None) -> dict[str, Any]: """Analyze an image submission using bits decomposition.""" diff --git a/aperisolve/analyzers/exiftool.py b/aperisolve/analyzers/exiftool.py index b749a8d..6e8f786 100644 --- a/aperisolve/analyzers/exiftool.py +++ b/aperisolve/analyzers/exiftool.py @@ -12,8 +12,8 @@ from .base_analyzer import SubprocessAnalyzer class ExiftoolAnalyzer(SubprocessAnalyzer): """Analyzer for exiftool.""" - def __init__(self, *args: Any) -> None: - super().__init__("exiftool", *args) + def __init__(self, input_img, output_dir: Any) -> None: + super().__init__("exiftool", input_img, output_dir) self.cmd = ["exiftool", "-a", "-u", "-g1", self.img] def process_output(self, stdout: str, stderr: str) -> dict[str, str]: diff --git a/aperisolve/analyzers/foremost.py b/aperisolve/analyzers/foremost.py index 8900c28..3d14448 100644 --- a/aperisolve/analyzers/foremost.py +++ b/aperisolve/analyzers/foremost.py @@ -12,8 +12,8 @@ from .base_analyzer import SubprocessAnalyzer class ForemostAnalyzer(SubprocessAnalyzer): """Analyzer for foremost.""" - def __init__(self, *args: Any) -> None: - super().__init__("foremost", *args, has_archive=True) + def __init__(self, input_img, output_dir: Any) -> None: + super().__init__("foremost", input_img, output_dir, has_archive=True) self.cmd = ["foremost", "-o", str(self.get_extracted_dir()), "-i", self.img] def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: diff --git a/aperisolve/analyzers/image_resize.py b/aperisolve/analyzers/image_resize.py index f9241b2..0192ff4 100644 --- a/aperisolve/analyzers/image_resize.py +++ b/aperisolve/analyzers/image_resize.py @@ -17,8 +17,8 @@ class ResizeAnalyzer(SubprocessAnalyzer): png: PNG - def __init__(self, *args: Any) -> None: - super().__init__("image_resize", *args, has_archive=True) + def __init__(self, input_img, output_dir: Any) -> None: + super().__init__("image_resize", input_img, output_dir, has_archive=True) def _lookup_crc(self) -> List[PNG]: """ diff --git a/aperisolve/analyzers/openstego.py b/aperisolve/analyzers/openstego.py index 26f6960..45d36a5 100644 --- a/aperisolve/analyzers/openstego.py +++ b/aperisolve/analyzers/openstego.py @@ -12,8 +12,8 @@ from .base_analyzer import SubprocessAnalyzer class OpenStegoAnalyzer(SubprocessAnalyzer): """Analyzer for openstego.""" - def __init__(self, *args: Any) -> None: - super().__init__("openstego", *args, has_archive=True) + def __init__(self, input_img, output_dir: Any) -> None: + super().__init__("openstego", input_img, output_dir, has_archive=True) self.algo = 0 def build_cmd(self, password: Optional[str] = None) -> list[str]: diff --git a/aperisolve/analyzers/outguess.py b/aperisolve/analyzers/outguess.py index cb8ec9d..a3750f3 100644 --- a/aperisolve/analyzers/outguess.py +++ b/aperisolve/analyzers/outguess.py @@ -12,8 +12,8 @@ from .base_analyzer import SubprocessAnalyzer class OutguessAnalyzer(SubprocessAnalyzer): """Analyzer for outguess.""" - def __init__(self, *args: Any) -> None: - super().__init__("outguess", *args, has_archive=True) + def __init__(self, input_img, output_dir: Any) -> None: + super().__init__("outguess", input_img, output_dir, has_archive=True) def build_cmd(self, password: Optional[str] = None) -> list[str]: extracted_dir = self.get_extracted_dir() diff --git a/aperisolve/analyzers/pngcheck.py b/aperisolve/analyzers/pngcheck.py index 03350e1..d987512 100644 --- a/aperisolve/analyzers/pngcheck.py +++ b/aperisolve/analyzers/pngcheck.py @@ -12,8 +12,8 @@ from .base_analyzer import SubprocessAnalyzer class PngcheckAnalyzer(SubprocessAnalyzer): """Analyzer for pngcheck.""" - def __init__(self, *args: Any) -> None: - super().__init__("pngcheck", *args) + def __init__(self, input_img, output_dir: Any) -> None: + super().__init__("pngcheck", input_img, output_dir) self.cmd = ["pngcheck", "-v", self.img] def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: diff --git a/aperisolve/analyzers/steghide.py b/aperisolve/analyzers/steghide.py index fb160fe..a79c0e1 100644 --- a/aperisolve/analyzers/steghide.py +++ b/aperisolve/analyzers/steghide.py @@ -13,8 +13,8 @@ from .base_analyzer import SubprocessAnalyzer class SteghideAnalyzer(SubprocessAnalyzer): """Analyzer for steghide.""" - def __init__(self, *args: Any) -> None: - super().__init__("steghide", *args, has_archive=True) + def __init__(self, input_img, output_dir: Any) -> None: + super().__init__("steghide", input_img, output_dir, has_archive=True) def build_cmd(self, password: Optional[str] = None) -> list[str]: if password is None: diff --git a/aperisolve/analyzers/strings.py b/aperisolve/analyzers/strings.py index 2285bbf..841b74c 100644 --- a/aperisolve/analyzers/strings.py +++ b/aperisolve/analyzers/strings.py @@ -12,8 +12,8 @@ from .base_analyzer import SubprocessAnalyzer class StringsAnalyzer(SubprocessAnalyzer): """Analyzer for strings.""" - def __init__(self, *args: Any) -> None: - super().__init__("strings", *args) + def __init__(self, input_img, output_dir: Any) -> None: + super().__init__("strings", input_img, output_dir) self.cmd = ["strings", self.img] diff --git a/aperisolve/analyzers/template_analyzer.py b/aperisolve/analyzers/template_analyzer.py index 70a743f..36e8997 100644 --- a/aperisolve/analyzers/template_analyzer.py +++ b/aperisolve/analyzers/template_analyzer.py @@ -12,8 +12,8 @@ from .base_analyzer import SubprocessAnalyzer class TemplateAnalyze(SubprocessAnalyzer): """Analyzer for .""" - def __init__(self, *args: Any) -> None: - super().__init__("", *args, has_archive=True) + def __init__(self, input_img, output_dir: Any) -> None: + super().__init__("", input_img, output_dir, has_archive=True) def build_cmd(self, password: Optional[str] = None) -> list[str]: if password: diff --git a/aperisolve/analyzers/zsteg.py b/aperisolve/analyzers/zsteg.py index d7de261..ffc8369 100644 --- a/aperisolve/analyzers/zsteg.py +++ b/aperisolve/analyzers/zsteg.py @@ -12,8 +12,8 @@ from .base_analyzer import SubprocessAnalyzer class ZstegAnalyzer(SubprocessAnalyzer): """Analyzer for zsteg.""" - def __init__(self, *args: Any) -> None: - super().__init__("zsteg", *args) + def __init__(self, input_img, output_dir: Any) -> None: + super().__init__("zsteg", input_img, output_dir) self.cmd = ["zsteg", self.img] def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: From 2ea69e2e46b2af52d2588b39f2619734dc25b512 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 21:04:03 +0100 Subject: [PATCH 068/102] fix mypy Path --- aperisolve/analyzers/decomposer.py | 2 +- aperisolve/analyzers/exiftool.py | 2 +- aperisolve/analyzers/foremost.py | 2 +- aperisolve/analyzers/image_resize.py | 2 +- aperisolve/analyzers/openstego.py | 2 +- aperisolve/analyzers/outguess.py | 2 +- aperisolve/analyzers/pngcheck.py | 2 +- aperisolve/analyzers/steghide.py | 2 +- aperisolve/analyzers/strings.py | 2 +- aperisolve/analyzers/template_analyzer.py | 2 +- aperisolve/analyzers/zsteg.py | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/aperisolve/analyzers/decomposer.py b/aperisolve/analyzers/decomposer.py index a9de9c6..9946c6e 100644 --- a/aperisolve/analyzers/decomposer.py +++ b/aperisolve/analyzers/decomposer.py @@ -15,7 +15,7 @@ from .base_analyzer import SubprocessAnalyzer class DecomposerAnalyzer(SubprocessAnalyzer): """Analyzer for bits decomposer.""" - def __init__(self, input_img, output_dir: Any) -> None: + def __init__(self, input_img: Path, output_dir: Path) -> None: super().__init__("decomposer", input_img, output_dir) def get_results(self, _: Optional[str] = None) -> dict[str, Any]: diff --git a/aperisolve/analyzers/exiftool.py b/aperisolve/analyzers/exiftool.py index 6e8f786..fbfd15e 100644 --- a/aperisolve/analyzers/exiftool.py +++ b/aperisolve/analyzers/exiftool.py @@ -12,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class ExiftoolAnalyzer(SubprocessAnalyzer): """Analyzer for exiftool.""" - def __init__(self, input_img, output_dir: Any) -> None: + def __init__(self, input_img: Path, output_dir: Path) -> None: super().__init__("exiftool", input_img, output_dir) self.cmd = ["exiftool", "-a", "-u", "-g1", self.img] diff --git a/aperisolve/analyzers/foremost.py b/aperisolve/analyzers/foremost.py index 3d14448..1167acc 100644 --- a/aperisolve/analyzers/foremost.py +++ b/aperisolve/analyzers/foremost.py @@ -12,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class ForemostAnalyzer(SubprocessAnalyzer): """Analyzer for foremost.""" - def __init__(self, input_img, output_dir: Any) -> None: + def __init__(self, input_img: Path, output_dir: Path) -> None: super().__init__("foremost", input_img, output_dir, has_archive=True) self.cmd = ["foremost", "-o", str(self.get_extracted_dir()), "-i", self.img] diff --git a/aperisolve/analyzers/image_resize.py b/aperisolve/analyzers/image_resize.py index 0192ff4..826d52d 100644 --- a/aperisolve/analyzers/image_resize.py +++ b/aperisolve/analyzers/image_resize.py @@ -17,7 +17,7 @@ class ResizeAnalyzer(SubprocessAnalyzer): png: PNG - def __init__(self, input_img, output_dir: Any) -> None: + def __init__(self, input_img: Path, output_dir: Path) -> None: super().__init__("image_resize", input_img, output_dir, has_archive=True) def _lookup_crc(self) -> List[PNG]: diff --git a/aperisolve/analyzers/openstego.py b/aperisolve/analyzers/openstego.py index 45d36a5..2588150 100644 --- a/aperisolve/analyzers/openstego.py +++ b/aperisolve/analyzers/openstego.py @@ -12,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class OpenStegoAnalyzer(SubprocessAnalyzer): """Analyzer for openstego.""" - def __init__(self, input_img, output_dir: Any) -> None: + def __init__(self, input_img: Path, output_dir: Path) -> None: super().__init__("openstego", input_img, output_dir, has_archive=True) self.algo = 0 diff --git a/aperisolve/analyzers/outguess.py b/aperisolve/analyzers/outguess.py index a3750f3..082d555 100644 --- a/aperisolve/analyzers/outguess.py +++ b/aperisolve/analyzers/outguess.py @@ -12,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class OutguessAnalyzer(SubprocessAnalyzer): """Analyzer for outguess.""" - def __init__(self, input_img, output_dir: Any) -> None: + def __init__(self, input_img: Path, output_dir: Path) -> None: super().__init__("outguess", input_img, output_dir, has_archive=True) def build_cmd(self, password: Optional[str] = None) -> list[str]: diff --git a/aperisolve/analyzers/pngcheck.py b/aperisolve/analyzers/pngcheck.py index d987512..b2e725b 100644 --- a/aperisolve/analyzers/pngcheck.py +++ b/aperisolve/analyzers/pngcheck.py @@ -12,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class PngcheckAnalyzer(SubprocessAnalyzer): """Analyzer for pngcheck.""" - def __init__(self, input_img, output_dir: Any) -> None: + def __init__(self, input_img: Path, output_dir: Path) -> None: super().__init__("pngcheck", input_img, output_dir) self.cmd = ["pngcheck", "-v", self.img] diff --git a/aperisolve/analyzers/steghide.py b/aperisolve/analyzers/steghide.py index a79c0e1..38b8961 100644 --- a/aperisolve/analyzers/steghide.py +++ b/aperisolve/analyzers/steghide.py @@ -13,7 +13,7 @@ from .base_analyzer import SubprocessAnalyzer class SteghideAnalyzer(SubprocessAnalyzer): """Analyzer for steghide.""" - def __init__(self, input_img, output_dir: Any) -> None: + def __init__(self, input_img: Path, output_dir: Path) -> None: super().__init__("steghide", input_img, output_dir, has_archive=True) def build_cmd(self, password: Optional[str] = None) -> list[str]: diff --git a/aperisolve/analyzers/strings.py b/aperisolve/analyzers/strings.py index 841b74c..1642c76 100644 --- a/aperisolve/analyzers/strings.py +++ b/aperisolve/analyzers/strings.py @@ -12,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class StringsAnalyzer(SubprocessAnalyzer): """Analyzer for strings.""" - def __init__(self, input_img, output_dir: Any) -> None: + def __init__(self, input_img: Path, output_dir: Path) -> None: super().__init__("strings", input_img, output_dir) self.cmd = ["strings", self.img] diff --git a/aperisolve/analyzers/template_analyzer.py b/aperisolve/analyzers/template_analyzer.py index 36e8997..94438ac 100644 --- a/aperisolve/analyzers/template_analyzer.py +++ b/aperisolve/analyzers/template_analyzer.py @@ -12,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class TemplateAnalyze(SubprocessAnalyzer): """Analyzer for .""" - def __init__(self, input_img, output_dir: Any) -> None: + def __init__(self, input_img: Path, output_dir: Path) -> None: super().__init__("", input_img, output_dir, has_archive=True) def build_cmd(self, password: Optional[str] = None) -> list[str]: diff --git a/aperisolve/analyzers/zsteg.py b/aperisolve/analyzers/zsteg.py index ffc8369..dfef6d0 100644 --- a/aperisolve/analyzers/zsteg.py +++ b/aperisolve/analyzers/zsteg.py @@ -12,7 +12,7 @@ from .base_analyzer import SubprocessAnalyzer class ZstegAnalyzer(SubprocessAnalyzer): """Analyzer for zsteg.""" - def __init__(self, input_img, output_dir: Any) -> None: + def __init__(self, input_img: Path, output_dir: Path) -> None: super().__init__("zsteg", input_img, output_dir) self.cmd = ["zsteg", self.img] From 3cdbfa6e1847f7e8076ecdde3735f8f263f1fa18 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 21:26:37 +0100 Subject: [PATCH 069/102] remove unused Any --- aperisolve/analyzers/binwalk.py | 2 +- aperisolve/analyzers/exiftool.py | 1 - aperisolve/analyzers/foremost.py | 1 - aperisolve/analyzers/outguess.py | 2 +- aperisolve/analyzers/pngcheck.py | 1 - aperisolve/analyzers/steghide.py | 2 +- aperisolve/analyzers/strings.py | 1 - aperisolve/analyzers/template_analyzer.py | 2 +- aperisolve/analyzers/zsteg.py | 1 - 9 files changed, 4 insertions(+), 9 deletions(-) diff --git a/aperisolve/analyzers/binwalk.py b/aperisolve/analyzers/binwalk.py index 7b4e8c0..f235b49 100644 --- a/aperisolve/analyzers/binwalk.py +++ b/aperisolve/analyzers/binwalk.py @@ -11,7 +11,7 @@ from .base_analyzer import SubprocessAnalyzer class BinwalkAnalyzer(SubprocessAnalyzer): """Analyzer for binwalk.""" - def __init__(self, input_img, output_dir) -> None: + def __init__(self, input_img: Path, output_dir: Path) -> None: super().__init__("binwalk", input_img, output_dir, has_archive=True) self.cmd = ["binwalk", "--matryoshka", "-e", self.img, "--run-as=root"] self.make_folder = False # Binwalk already create its own folder diff --git a/aperisolve/analyzers/exiftool.py b/aperisolve/analyzers/exiftool.py index fbfd15e..d35d216 100644 --- a/aperisolve/analyzers/exiftool.py +++ b/aperisolve/analyzers/exiftool.py @@ -4,7 +4,6 @@ """Exiftool Analyzer for Image Submissions.""" from pathlib import Path -from typing import Any from .base_analyzer import SubprocessAnalyzer diff --git a/aperisolve/analyzers/foremost.py b/aperisolve/analyzers/foremost.py index 1167acc..85a667b 100644 --- a/aperisolve/analyzers/foremost.py +++ b/aperisolve/analyzers/foremost.py @@ -4,7 +4,6 @@ """Foremost Analyzer for Image Submissions.""" from pathlib import Path -from typing import Any from .base_analyzer import SubprocessAnalyzer diff --git a/aperisolve/analyzers/outguess.py b/aperisolve/analyzers/outguess.py index 082d555..e99d996 100644 --- a/aperisolve/analyzers/outguess.py +++ b/aperisolve/analyzers/outguess.py @@ -4,7 +4,7 @@ """Outguess Analyzer for Image Submissions.""" from pathlib import Path -from typing import Any, Optional +from typing import Optional from .base_analyzer import SubprocessAnalyzer diff --git a/aperisolve/analyzers/pngcheck.py b/aperisolve/analyzers/pngcheck.py index b2e725b..530c5d6 100644 --- a/aperisolve/analyzers/pngcheck.py +++ b/aperisolve/analyzers/pngcheck.py @@ -4,7 +4,6 @@ """Pngcheck Analyzer for Image Submissions.""" from pathlib import Path -from typing import Any from .base_analyzer import SubprocessAnalyzer diff --git a/aperisolve/analyzers/steghide.py b/aperisolve/analyzers/steghide.py index 38b8961..aa15f52 100644 --- a/aperisolve/analyzers/steghide.py +++ b/aperisolve/analyzers/steghide.py @@ -5,7 +5,7 @@ import re from pathlib import Path -from typing import Any, Optional +from typing import Optional from .base_analyzer import SubprocessAnalyzer diff --git a/aperisolve/analyzers/strings.py b/aperisolve/analyzers/strings.py index 1642c76..4d25a58 100644 --- a/aperisolve/analyzers/strings.py +++ b/aperisolve/analyzers/strings.py @@ -4,7 +4,6 @@ """Strings Analyzer for Image Submissions.""" from pathlib import Path -from typing import Any from .base_analyzer import SubprocessAnalyzer diff --git a/aperisolve/analyzers/template_analyzer.py b/aperisolve/analyzers/template_analyzer.py index 94438ac..7785397 100644 --- a/aperisolve/analyzers/template_analyzer.py +++ b/aperisolve/analyzers/template_analyzer.py @@ -4,7 +4,7 @@ """ Analyzer for Image Submissions.""" from pathlib import Path -from typing import Any, Optional +from typing import Optional from .base_analyzer import SubprocessAnalyzer diff --git a/aperisolve/analyzers/zsteg.py b/aperisolve/analyzers/zsteg.py index dfef6d0..3d111ad 100644 --- a/aperisolve/analyzers/zsteg.py +++ b/aperisolve/analyzers/zsteg.py @@ -4,7 +4,6 @@ """Strings Analyzer for Image Submissions.""" from pathlib import Path -from typing import Any from .base_analyzer import SubprocessAnalyzer From 0b790b02c8e73e6face97e8dd69518d9c6a5e400 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Fri, 2 Jan 2026 23:33:18 +0100 Subject: [PATCH 070/102] fix copilot reviews --- aperisolve/analyzers/binwalk.py | 2 +- aperisolve/analyzers/image_resize.py | 4 ++-- aperisolve/analyzers/steghide.py | 3 ++- aperisolve/analyzers/template_analyzer.py | 4 ++-- aperisolve/analyzers/zsteg.py | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/aperisolve/analyzers/binwalk.py b/aperisolve/analyzers/binwalk.py index f235b49..caf755a 100644 --- a/aperisolve/analyzers/binwalk.py +++ b/aperisolve/analyzers/binwalk.py @@ -20,7 +20,7 @@ class BinwalkAnalyzer(SubprocessAnalyzer): return self.output_dir / f"_{self.input_img.name}.extracted" def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: - return len(stderr) > 0 and not zip_exist and False + return len(stderr) > 0 and not zip_exist def analyze_binwalk(input_img: Path, output_dir: Path) -> None: diff --git a/aperisolve/analyzers/image_resize.py b/aperisolve/analyzers/image_resize.py index 826d52d..4b4a12a 100644 --- a/aperisolve/analyzers/image_resize.py +++ b/aperisolve/analyzers/image_resize.py @@ -96,7 +96,7 @@ class ResizeAnalyzer(SubprocessAnalyzer): found, it recovers and saves PNG images with the matched dimensions. Returns: - None + dict[str, Any]: A dictionary containing the analysis status, any error message, Raises: Handles exceptions gracefully by updating output data with error status. @@ -165,6 +165,6 @@ class ResizeAnalyzer(SubprocessAnalyzer): def analyze_image_resize(input_img: Path, output_dir: Path) -> None: - """Analyze an image submission using outguess.""" + """Analyze an image submission using ResizeAnalyzer.""" analyzer = ResizeAnalyzer(input_img, output_dir) analyzer.analyze() diff --git a/aperisolve/analyzers/steghide.py b/aperisolve/analyzers/steghide.py index aa15f52..c1785c2 100644 --- a/aperisolve/analyzers/steghide.py +++ b/aperisolve/analyzers/steghide.py @@ -30,7 +30,8 @@ class SteghideAnalyzer(SubprocessAnalyzer): match = re.search(r'embedded file "([^"]+)"', data.stdout) assert match is not None hidden_file = match.group(1) - outfile = str(self.get_extracted_dir() / hidden_file) + safe_hidden_file = Path(hidden_file).name + outfile = str(self.get_extracted_dir() / safe_hidden_file) cmd = ["steghide", "extract", "-sf", self.img, "-xf", outfile, "-p", password] return cmd diff --git a/aperisolve/analyzers/template_analyzer.py b/aperisolve/analyzers/template_analyzer.py index 7785397..cb5306a 100644 --- a/aperisolve/analyzers/template_analyzer.py +++ b/aperisolve/analyzers/template_analyzer.py @@ -9,7 +9,7 @@ from typing import Optional from .base_analyzer import SubprocessAnalyzer -class TemplateAnalyze(SubprocessAnalyzer): +class TemplateAnalyzer(SubprocessAnalyzer): """Analyzer for .""" def __init__(self, input_img: Path, output_dir: Path) -> None: @@ -23,7 +23,7 @@ class TemplateAnalyze(SubprocessAnalyzer): def analyze_template(input_img: Path, output_dir: Path, password: Optional[str] = None) -> None: """Analyze an image submission using .""" - analyzer = TemplateAnalyze(input_img, output_dir) + analyzer = TemplateAnalyzer(input_img, output_dir) if password: analyzer.analyze(password) else: diff --git a/aperisolve/analyzers/zsteg.py b/aperisolve/analyzers/zsteg.py index 3d111ad..df67ef4 100644 --- a/aperisolve/analyzers/zsteg.py +++ b/aperisolve/analyzers/zsteg.py @@ -1,7 +1,7 @@ # flake8: noqa: E203,E501,W503 # pylint: disable=C0413,W0718,R0903,R0801 # mypy: disable-error-code=unused-awaitable -"""Strings Analyzer for Image Submissions.""" +"""Zsteg Analyzer for Image Submissions.""" from pathlib import Path From 66211c29e652157df3f9fae690cea15907380ba0 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Sat, 3 Jan 2026 00:09:59 +0100 Subject: [PATCH 071/102] fix 109 --- aperisolve/analyzers/base_analyzer.py | 1 + aperisolve/analyzers/decomposer.py | 111 +++++++++++++------------- aperisolve/analyzers/image_resize.py | 75 +++++++++-------- aperisolve/analyzers/openstego.py | 1 + aperisolve/workers.py | 20 ++++- 5 files changed, 109 insertions(+), 99 deletions(-) diff --git a/aperisolve/analyzers/base_analyzer.py b/aperisolve/analyzers/base_analyzer.py index 5f20a05..2ead24a 100644 --- a/aperisolve/analyzers/base_analyzer.py +++ b/aperisolve/analyzers/base_analyzer.py @@ -161,6 +161,7 @@ class SubprocessAnalyzer(ABC): self.update_result(result) except Exception as e: self.update_result({"status": "error", "error": str(e)}) + raise def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: """Check if the result is an error.""" diff --git a/aperisolve/analyzers/decomposer.py b/aperisolve/analyzers/decomposer.py index 9946c6e..a3ff02a 100644 --- a/aperisolve/analyzers/decomposer.py +++ b/aperisolve/analyzers/decomposer.py @@ -20,73 +20,70 @@ class DecomposerAnalyzer(SubprocessAnalyzer): def get_results(self, _: Optional[str] = None) -> dict[str, Any]: """Analyze an image submission using bits decomposition.""" - try: - img = Image.open(self.input_img) - converted = False + img = Image.open(self.input_img) + converted = False - if img.mode == "P": - img = img.convert("RGB") - converted = True + if img.mode == "P": + img = img.convert("RGB") + converted = True - img_np = np.array(img) + img_np = np.array(img) - # Handle grayscale, RGB, RGBA, etc. - if len(img_np.shape) == 2: - channels = 1 - else: - channels = img_np.shape[2] + # Handle grayscale, RGB, RGBA, etc. + if len(img_np.shape) == 2: + channels = 1 + else: + channels = img_np.shape[2] - if channels > 1: - channel_names = ["Red", "Green", "Blue", "Alpha"] - else: - channel_names = ["Grayscale"] + if channels > 1: + channel_names = ["Red", "Green", "Blue", "Alpha"] + else: + channel_names = ["Grayscale"] - image_json = {} + image_json = {} - # Add Superimposed RGB bit planes - if channels >= 3: - superimposed_json = [] - for bit in range(8): - bit_mask = 1 << bit - rgb_planes = [((img_np[..., c] & bit_mask) >> bit) * 255 for c in range(3)] - rgb_image = np.stack(rgb_planes, axis=-1).astype(np.uint8) - rgb_img = Image.fromarray(rgb_image, mode="RGB") - img_name = f"superimposed_bit_{bit}.png" - out_path = self.output_dir / img_name - dl_path = Path(self.output_dir.name) / img_name - superimposed_json.append("/image/" + str(dl_path)) - rgb_img.save(out_path) + # Add Superimposed RGB bit planes + if channels >= 3: + superimposed_json = [] + for bit in range(8): + bit_mask = 1 << bit + rgb_planes = [((img_np[..., c] & bit_mask) >> bit) * 255 for c in range(3)] + rgb_image = np.stack(rgb_planes, axis=-1).astype(np.uint8) + rgb_img = Image.fromarray(rgb_image, mode="RGB") + img_name = f"superimposed_bit_{bit}.png" + out_path = self.output_dir / img_name + dl_path = Path(self.output_dir.name) / img_name + superimposed_json.append("/image/" + str(dl_path)) + rgb_img.save(out_path) - image_json["Superimposed"] = superimposed_json + image_json["Superimposed"] = superimposed_json - # Process individual channels - for c in range(channels): - channel_data = img_np[..., c] if channels > 1 else img_np - channel_label = channel_names[c] - channel_json = [] - for bit in range(8): - bit_mask = 1 << bit - bit_plane = ((channel_data & bit_mask) >> bit) * 255 - bit_img = Image.fromarray(bit_plane.astype(np.uint8), mode="L") - img_name = f"{channel_label}_bit_{bit}.png" - out_path = self.output_dir / img_name - dl_path = Path(self.output_dir.name) / img_name - channel_json.append("/image/" + str(dl_path)) - bit_img.save(out_path) - image_json[channel_label] = channel_json + # Process individual channels + for c in range(channels): + channel_data = img_np[..., c] if channels > 1 else img_np + channel_label = channel_names[c] + channel_json = [] + for bit in range(8): + bit_mask = 1 << bit + bit_plane = ((channel_data & bit_mask) >> bit) * 255 + bit_img = Image.fromarray(bit_plane.astype(np.uint8), mode="L") + img_name = f"{channel_label}_bit_{bit}.png" + out_path = self.output_dir / img_name + dl_path = Path(self.output_dir.name) / img_name + channel_json.append("/image/" + str(dl_path)) + bit_img.save(out_path) + image_json[channel_label] = channel_json - output = { - "status": "ok", - "images": image_json, - } - if converted: - output["note"] = ( - "Image contains a color palette and was converted to RGB for processing." - ) + output = { + "status": "ok", + "images": image_json, + } + if converted: + output["note"] = ( + "Image contains a color palette and was converted to RGB for processing." + ) - return output - except Exception as e: - return {"status": "error", "error": str(e)} + return output def analyze_decomposer(input_img: Path, output_dir: Path) -> None: diff --git a/aperisolve/analyzers/image_resize.py b/aperisolve/analyzers/image_resize.py index 4b4a12a..c853350 100644 --- a/aperisolve/analyzers/image_resize.py +++ b/aperisolve/analyzers/image_resize.py @@ -115,53 +115,50 @@ class ResizeAnalyzer(SubprocessAnalyzer): """ logs = [] - try: - with self.input_img.open("rb") as image: - img_bytes = bytearray(image.read()) - try: - self.png = PNG.from_bytearray(img_bytes) - except ValueError: - return { - "status": "error", - "error": "IHDR chunk is not the first chunk, or PNG has invalid structure.", - } - - # Already correct - if self.png.crc == self.png.compute_crc(): - return { - "status": "ok", - "note": f"PNG is already valid with dimensions " - f"{self.png.width}x{self.png.height} and crc 0x{self.png.crc:08x}.", - } - - saved_img_urls = [] - - # No match found - if not (db_matches := self._lookup_crc()) and not ( - db_matches := self._search_height_crc() - ): + with self.input_img.open("rb") as image: + img_bytes = bytearray(image.read()) + try: + self.png = PNG.from_bytearray(img_bytes) + except ValueError: return { "status": "error", - "error": "Failure: No matching dimensions found.", + "error": "IHDR chunk is not the first chunk, or PNG has invalid structure.", } - # Match found, saving image(s) - logs.append(f"Target CRC found: 0x{self.png.crc:08x}") - self.output_dir.mkdir(parents=True, exist_ok=True) - for png_match in db_matches: - img_name = self._write_recovered_png(img_bytes, png_match) - logs.append(f"Image saved: {img_name}") - # Append the formatted URL to our list - saved_img_urls.append("/image/" + str(Path(self.output_dir.name) / img_name)) - + # Already correct + if self.png.crc == self.png.compute_crc(): return { "status": "ok", - "output": logs, - "png_images": saved_img_urls, + "note": f"PNG is already valid with dimensions " + f"{self.png.width}x{self.png.height} and crc 0x{self.png.crc:08x}.", } - except Exception as e: - return {"status": "error", "error": str(e)} + saved_img_urls = [] + + # No match found + if not (db_matches := self._lookup_crc()) and not ( + db_matches := self._search_height_crc() + ): + return { + "status": "error", + "error": "Failure: No matching dimensions found.", + } + + # Match found, saving image(s) + logs.append(f"Target CRC found: 0x{self.png.crc:08x}") + self.output_dir.mkdir(parents=True, exist_ok=True) + for png_match in db_matches: + img_name = self._write_recovered_png(img_bytes, png_match) + logs.append(f"Image saved: {img_name}") + # Append the formatted URL to our list + saved_img_urls.append("/image/" + str(Path(self.output_dir.name) / img_name)) + + return { + "status": "ok", + "output": logs, + "png_images": saved_img_urls, + } + def analyze_image_resize(input_img: Path, output_dir: Path) -> None: diff --git a/aperisolve/analyzers/openstego.py b/aperisolve/analyzers/openstego.py index 2588150..559482a 100644 --- a/aperisolve/analyzers/openstego.py +++ b/aperisolve/analyzers/openstego.py @@ -44,6 +44,7 @@ class OpenStegoAnalyzer(SubprocessAnalyzer): self.update_result(result) except Exception as e: self.update_result({"status": "error", "error": str(e)}) + raise def process_output(self, stdout: str, stderr: str) -> str | list[str] | dict[str, str]: """Process the stdout/stderr.""" diff --git a/aperisolve/workers.py b/aperisolve/workers.py index 8ab66ef..87a499c 100644 --- a/aperisolve/workers.py +++ b/aperisolve/workers.py @@ -77,12 +77,26 @@ def analyze_image(submission_hash: str) -> None: def run_analyzer(analyzer_func: Any, *args: Any) -> None: """Run an analyzer function in a separate thread.""" + analyzer_name = analyzer_func.__name__.replace("analyze_", "") try: analyzer_func(*args) except Exception as e: - sentry_sdk.capture_exception(e) - print(f"Error in {analyzer_func.__name__}: {e}") - sentry_sdk.capture_exception(e) + print(f"Error in {analyzer_name}: {e}") + with sentry_sdk.push_scope() as scope: + scope.set_tag("analyzer", analyzer_name) + scope.set_tag("submission_hash", submission_hash) + scope.set_context( + "analyzer_info", + { + "tool": analyzer_name, + "image_path": str(img_path), + "result_path": str(result_path), + "filename": submission.filename, + "deep_analysis": submission.deep_analysis, + }, + ) + scope.fingerprint = ["analyzer-error", analyzer_name] + sentry_sdk.capture_exception(e) analyzers = [ (analyze_binwalk, img_path, result_path), From 8c1529b6c9c941715f6a9c9f1c7e7987d0556de6 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Sat, 3 Jan 2026 00:51:58 +0100 Subject: [PATCH 072/102] black --- aperisolve/analyzers/image_resize.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/aperisolve/analyzers/image_resize.py b/aperisolve/analyzers/image_resize.py index c853350..b19447e 100644 --- a/aperisolve/analyzers/image_resize.py +++ b/aperisolve/analyzers/image_resize.py @@ -136,9 +136,7 @@ class ResizeAnalyzer(SubprocessAnalyzer): saved_img_urls = [] # No match found - if not (db_matches := self._lookup_crc()) and not ( - db_matches := self._search_height_crc() - ): + if not (db_matches := self._lookup_crc()) and not (db_matches := self._search_height_crc()): return { "status": "error", "error": "Failure: No matching dimensions found.", @@ -160,7 +158,6 @@ class ResizeAnalyzer(SubprocessAnalyzer): } - def analyze_image_resize(input_img: Path, output_dir: Path) -> None: """Analyze an image submission using ResizeAnalyzer.""" analyzer = ResizeAnalyzer(input_img, output_dir) From b3605047ad145a8859c1ae329351dc6ef96a797a Mon Sep 17 00:00:00 2001 From: Zeecka Date: Sat, 3 Jan 2026 01:44:56 +0100 Subject: [PATCH 073/102] add CONTRIBUTING --- CONTRIBUTING.md | 186 +++++++++++++++++++++++++++++++++++ README.md | 23 +---- docs/adding-analyzer.md | 211 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 401 insertions(+), 19 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 docs/adding-analyzer.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..84468c0 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,186 @@ +# Contributing to Aperi'Solve + +Thank you for your interest in contributing! We welcome all contributions, whether it's bug fixes, new features, or documentation improvements. + +## Getting Started + +> [!TIP] +> New to the project? Adding a new analyzer is a great first contribution! Check out our [Adding a New Analyzer Guide](docs/adding-analyzer.md). + +### Quick Steps + +1. **Fork the repository** +2. **Create a feature branch** (`git checkout -b feature/amazing-feature`) +3. **Make your changes** +4. **Test your changes** +5. **Commit with clear messages** (`git commit -m 'Add amazing feature'`) +6. **Push to your fork** (`git push origin feature/amazing-feature`) +7. **Open a Pull Request** describing your changes + +## What Can You Contribute? + +### 🔧 Adding New Analyzers + +Want to add support for a new steganography or forensics tool? We'd love that! + +> [!NOTE] +> Check out our detailed tutorial: [Adding a New Analyzer](docs/adding-analyzer.md) + +**Quick checklist for new analyzers:** +- [ ] Create analyzer file in `analyzers/` +- [ ] Extend `SubprocessAnalyzer` base class +- [ ] Register in `workers.py` +- [ ] Add tool to Docker setup if needed +- [ ] Test with sample images +- [ ] Update documentation + +### 🐛 Bug Fixes + +Found a bug? Please: +- Check if it's already reported in [Issues](../../issues) +- If not, open a new issue with reproduction steps +- Feel free to submit a PR with the fix! + +### 📚 Documentation + +Help us improve: +- Fix typos or unclear explanations +- Add examples and use cases +- Improve installation instructions +- Translate documentation + +### 🎨 UI/UX Improvements + +Contributions to the web interface are welcome: +- Better error messages +- Improved result presentation +- Mobile responsiveness +- Accessibility improvements + +## Code Style & Quality + +> [!IMPORTANT] +> Please follow the code style and run linters before submitting. + +The project adheres to: + +- **Black** (line length 100) +- **Isort** (profile black) +- **Flake8** (ignoring E203, E501, W503) +- **Pylint** (ignoring W0718, R0903, R0801) +- **Mypy** (ignoring unused-awaitable) + +See [.pre-commit-config.yaml](.pre-commit-config.yaml) for exact configuration. + +### Running Linters Locally + +> [!TIP] +> Install pre-commit hooks to automatically check your code before commits: +> ```bash +> pip install pre-commit +> pre-commit install +> ``` + +**Manual checks:** +```bash +# Format code +black . --line-length 100 +isort . --profile black + +# Check for issues +flake8 . +pylint **/*.py +mypy . +``` + +> [!NOTE] +> CI will automatically run these checks on every PR. Make sure they pass! + +## Pull Request Guidelines + +Use clear, descriptive titles: +- ✅ `Add stegdetect analyzer` +- ✅ `Fix binwalk extraction error handling` +- ✅ `Update documentation for password-protected tools` +- ❌ `Update code` +- ❌ `Fix bug` + + +### Commit Messages + +Write clear, concise commit messages: +```bash +# Good +git commit -m "Add zsteg analyzer for PNG steganography detection" +git commit -m "Fix foremost error detection when no files extracted" + +# Not so good +git commit -m "updates" +git commit -m "fix" +``` + +## Adding New Dependencies + +> [!CAUTION] +> Adding new dependencies requires careful consideration. + +If your contribution needs new dependencies: + +1. **Python packages**: Add to `requirements.txt` +2. **System tools**: Add to `Dockerfile` +3. **Explain why** in your PR description +4. **Keep dependencies minimal** - avoid adding large libraries for small features + +## Docker and Tool Installation + +When adding a new analyzer that requires a new tool: + +```dockerfile +# In Dockerfile +RUN apt-get update && apt-get install -y \ + your-new-tool \ + && rm -rf /var/lib/apt/lists/* +``` + +Or for tools requiring compilation: +```dockerfile +RUN git clone https://github.com/tool/repo.git /tmp/tool && \ + cd /tmp/tool && \ + make && make install && \ + rm -rf /tmp/tool +``` + +> [!NOTE] +> Test the Docker build locally before submitting! + +## Code Review Process + +1. **Automated checks** run first (linters, tests) +2. **Maintainer review** - we'll provide feedback +3. **Address feedback** - update your PR as needed +4. **Approval & merge** - once everything looks good! + +> [!TIP] +> Don't worry if you need to make changes - it's a normal part of the process! + +## Need Help? + +- 💬 **Questions?** Open a [Discussion](../../discussions) +- 🐛 **Found a bug?** Open an [Issue](../../issues) +- 📧 **Email:** [Your contact email] +- 💡 **Ideas?** We'd love to hear them in Discussions! + +## Recognition + +> [!NOTE] +> All contributors will be recognized in our [Contributors](../../graphs/contributors) page and in release notes! + +## License + +By contributing, you agree that your contributions will be licensed under the same license as the project (see [LICENSE](LICENSE)). + +--- + +**Thank you for contributing! 🎉** + +Your efforts help make this tool better for the entire security and CTF community! diff --git a/README.md b/README.md index b84a9ea..d30b0e6 100644 --- a/README.md +++ b/README.md @@ -191,32 +191,17 @@ This separation keeps heavy tools (binwalk, foremost, zsteg, etc.) isolated and ## Roadmap - [ ] **[Bug]** Duplicate key value violates unique constraint "submission_pkey" +- [ ] **[Enhancement]** Server continuous deployment - [ ] **[Feature]** Zsteg: full extraction (--all) and download of discovered files (mp3, etc.) - [ ] **[Feature]** Mobile-friendly UI / test on mobile - [ ] **[Feature]** i18n (internationalization) - [ ] **[Feature]** Rootless / unprivileged analyzers -- [ ] **[Feature]** Improve analyzer sandboxing (e.g., per-analyzer containers) + +See [Issues](../../issues) for more! ## Contributing -Contributions are welcome! Please: - -1. Fork the repository -2. Create a feature branch -3. Open a pull request describing your change - -> [!IMPORTANT] -> Please follow the code style and run linters before submitting. The project adheres to: -> -> - Black (line length 100) -> - Isort (profile black) -> - Flake8 (ignoring E203, E501, W503) -> - Pylint (ignoring W0718, R0903, R0801) -> - Mypy (ignoring unused-awaitable) -> -> See [.pre-commit-config.yaml](.pre-commit-config.yaml) file. - -CI will run these checks on each PR. +See [CONTRIBUTING.md](CONTRIBUTING.md). ## Security diff --git a/docs/adding-analyzer.md b/docs/adding-analyzer.md new file mode 100644 index 0000000..0c6d0ca --- /dev/null +++ b/docs/adding-analyzer.md @@ -0,0 +1,211 @@ +# Adding a New Analyzer + +This guide shows you how to add a new image analysis tool to the system. + +## Quick Start + +1. Create a new file in `analyzers/` from template file `template_analyzer.py`. +2. Register it in `workers.py` + +## Basic Template + +```python +from pathlib import Path +from typing import Optional +from .base_analyzer import SubprocessAnalyzer + +class MyToolAnalyzer(SubprocessAnalyzer): + def __init__(self, input_img: Path, output_dir: Path) -> None: + super().__init__("mytool", input_img, output_dir) + self.cmd = ["mytool", self.img] + +def analyze_mytool(input_img: Path, output_dir: Path) -> None: + analyzer = MyToolAnalyzer(input_img, output_dir) + analyzer.analyze() +``` + +You can also see `template_analyzer.py`. + +## Common Scenarios + +### Simple Text Output Tool + +Like `strings` or `exiftool` - just reads the file and outputs text: + +```python +class StringsAnalyzer(SubprocessAnalyzer): + def __init__(self, input_img: Path, output_dir: Path) -> None: + super().__init__("strings", input_img, output_dir) + self.cmd = ["strings", self.img] +``` + +### Tool That Extracts Files + +Like `binwalk` or `foremost` - extracts files that get zipped for download: + +```python +class ForemostAnalyzer(SubprocessAnalyzer): + def __init__(self, input_img: Path, output_dir: Path) -> None: + super().__init__("foremost", input_img, output_dir, has_archive=True) + self.cmd = ["foremost", "-o", str(self.get_extracted_dir()), "-i", self.img] +``` + +Note the `has_archive=True` parameter. + +### Tool With Password Support + +Like `steghide` or `openstego`: + +```python +class OutguessAnalyzer(SubprocessAnalyzer): + def __init__(self, input_img: Path, output_dir: Path) -> None: + super().__init__("outguess", input_img, output_dir, has_archive=True) + + def build_cmd(self, password: Optional[str] = None) -> list[str]: + out = str(self.get_extracted_dir() / "outguess.data") + if password: + return ["outguess", "-k", password, "-r", self.img, out] + return ["outguess", "-r", self.img, out] + +def analyze_outguess(input_img: Path, output_dir: Path, password: Optional[str] = None) -> None: + analyzer = OutguessAnalyzer(input_img, output_dir) + if password: + analyzer.analyze(password) + else: + analyzer.analyze() +``` + +## Customizing Behavior + +### Custom Error Detection + +Override `is_error()` to define what counts as an error: + +```python +def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: + # Only error if no files were extracted + return len(stderr) > 0 and not zip_exist +``` + +### Custom Output Formatting + +Override `process_output()` to format the results: + +```python +def process_output(self, stdout: str, stderr: str) -> dict[str, str]: + # Parse into key-value pairs + metadata = {} + for line in stdout.split("\n"): + if ":" in line: + key, value = line.split(":", 1) + metadata[key.strip()] = value.strip() + return metadata +``` + +### User-Friendly Error Messages + +Override `process_error()` to provide helpful messages: + +```python +def process_error(self, stdout: str, stderr: str) -> str: + if "not supported" in stdout: + return "This file format is not supported (PNG only)" + return stderr +``` + +## Registering Your Analyzer + +Add it to `workers.py` in the `analyze_image` function: + +```python +# Import at the top +from .analyzers.mytool import analyze_mytool + +# Add to analyzers list +analyzers = [ + (analyze_binwalk, img_path, result_path), + (analyze_exiftool, img_path, result_path), + # ... other analyzers ... + (analyze_mytool, img_path, result_path), +] + +# If it needs a password, pass submission.password: +# (analyze_mytool, img_path, result_path, submission.password), + +# If only for deep analysis, add it conditionally: +if submission.deep_analysis: + analyzers.extend([ + (analyze_mytool, img_path, result_path), + ]) +``` + +## Available Methods to Override + +| Method | Purpose | Default Behavior | +|--------|---------|------------------| +| `build_cmd()` | Build command to run | Uses `self.cmd` | +| `is_error()` | Check if result is error | Returns `True` if stderr exists | +| `process_output()` | Format success output | Returns stdout as list of lines | +| `process_error()` | Format error message | Returns stderr as-is | +| `process_note()` | Add informational note | Returns `None` | +| `get_extracted_dir()` | Set extraction directory | Returns `output_dir/toolname/` | + +## Output Format + +Your analyzer produces JSON in `results.json`: + +**Success:** +```json +{ + "mytool": { + "status": "ok", + "output": "Tool output here", + "download": "/download/hash/mytool" + } +} +``` + +**Error:** +```json +{ + "mytool": { + "status": "error", + "error": "Error message" + } +} +``` + +## Tips + +- Use `self.img` in commands (relative path), not `self.input_img` +- Set `has_archive=True` if your tool extracts files +- Tools run in parallel threads - no need to worry about concurrency +- The base class handles timeouts (10 minutes default) +- Extracted files are automatically zipped into `.7z` archives +- All exceptions are caught and logged to Sentry + +## Example: Complete Analyzer + +```python +from pathlib import Path +from .base_analyzer import SubprocessAnalyzer + +class PngcheckAnalyzer(SubprocessAnalyzer): + def __init__(self, input_img: Path, output_dir: Path) -> None: + super().__init__("pngcheck", input_img, output_dir) + self.cmd = ["pngcheck", "-v", self.img] + + def is_error(self, returncode: int, stdout: str, stderr: str, zip_exist: bool) -> bool: + return "this is neither a PNG or JNG image" in stdout + + def process_error(self, stdout: str, stderr: str) -> str: + if "neither a PNG or JNG" in stdout: + return "File format not supported (PNG/JNG only)" + return stdout + +def analyze_pngcheck(input_img: Path, output_dir: Path) -> None: + analyzer = PngcheckAnalyzer(input_img, output_dir) + analyzer.analyze() +``` + +That's it! Your analyzer will now run in parallel with the others and results will appear in the web interface. From 604007788d4fbee445e76189659b4a260f7f5f85 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Sat, 3 Jan 2026 02:02:50 +0100 Subject: [PATCH 074/102] Fix readme --- .env.example | 2 +- README.md | 87 +++++++++++++---------------------------- docs/adding-analyzer.md | 2 + 3 files changed, 31 insertions(+), 60 deletions(-) diff --git a/.env.example b/.env.example index 38a8106..150aee9 100644 --- a/.env.example +++ b/.env.example @@ -3,7 +3,7 @@ MAX_CONTENT_LENGTH=1048576 MAX_PENDING_TIME=600 MAX_STORE_TIME=259200 CLEAR_AT_RESTART=1 -SKIP_IHDR_FILL=1 +SKIP_IHDR_FILL=0 # Flask configuration FLASK_DEBUG=0 diff --git a/README.md b/README.md index d30b0e6..d4be106 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ Table of contents - [Key features](#key-features) - [Quick start (Docker)](#quick-start-docker) - [Development](#development) -- [Adding a new analyzer](#adding-a-new-analyzer) - [Configuration & environment variables](#configuration--environment-variables) - [Architecture](#architecture) - [Troubleshooting & tips](#troubleshooting--tips) @@ -64,6 +63,7 @@ Clone and start the full stack (production-like): ```bash git clone https://github.com/Zeecka/AperiSolve.git cd AperiSolve +cp .env.example .env docker compose up -d ``` @@ -103,69 +103,38 @@ docker cp aperisolve-web:/aperisolve/results/filename.ext /path/to/backup/filena > rm -rf aperisolve/results > ``` -### Adding a new analyzer - -Adding a custom analyzer is straightforward: - -1. Create your analyzer file: - - - Copy `aperisolve/analyzers/template_analyzer.py` -> `aperisolve/analyzers/myanalyzer.py` - - Implement function signature similar to: - - ```python - # aperisolve/analyzers/myanalyzer.py - def analyze_myanalyzer(image_path: str, results_dir: str) -> dict: - """ - Perform analysis on image located at image_path. - Produce outputs in results_dir and return a result dict (json-serializable). - """ - # your analysis logic here - return {"name": "myanalyzer", "status": "ok", "outputs": [...]} - ``` - -2. Register the analyzer in the worker pipeline: - - - Edit `aperisolve/workers.py`: - - ```python - # import - from .analyzers.myanalyzer import analyze_myanalyzer - - # add to analyzers list (preserve the order) - analyzers = [ - analyze_zsteg, - analyze_steghide, - # ... - analyze_myanalyzer, - ] - ``` - -3. Add your analyzer to the UI order: - - - Edit `aperisolve/static/js/aperisolve.js` and append `myanalyzer` to `TOOL_ORDER` so it appears in the frontend. - -4. Test locally: run the worker and submit jobs to ensure outputs are produced and displayed. - -> [!TIP] -> -> - Keep analyzers idempotent and write outputs to the provided `results_dir`. -> - Return structured JSON so the frontend can render links/downloads automatically. - ## Configuration & environment variables -Typical services: +The application can be configured with the `.env` file: -- Web app (Flask) -- Workers (Python) -- Redis (RQ) -- PostgreSQL +```shell +# Application configuration +MAX_CONTENT_LENGTH=1048576 # Max uploaded image size (bytes) +MAX_PENDING_TIME=600 # Timeout for analyzer (seconds) +MAX_STORE_TIME=259200 # Delay until deleted from server +CLEAR_AT_RESTART=1 # Reset database and results at application restart/crash +SKIP_IHDR_FILL=0 # Skip IHDR database fill (gain times at startup but reduce results) -Main environment variables (examples): +# Flask configuration +FLASK_DEBUG=0 +FLASK_ENV=development -- DATABASE_URL=postgresql://aperiuser:password@postgres:5432/aperisolve -- REDIS_URL=redis://redis:6379/0 -- SECRET_KEY=change_me -- FLASK_ENV=production/development +# Redis configuration +RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0 + +# Database configuration +POSTGRES_DB=aperisolve +POSTGRES_USER=aperiuser +POSTGRES_PASSWORD=aperipass +DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve + +# Sentry configuration +SENTRY_DSN=https://your-dsn@sentry.io/project-id +SENTRY_TRACES_SAMPLE_RATE=0.1 +SENTRY_PROFILES_SAMPLE_RATE=0.1 +SENTRY_RELEASE=1.0.0 +SENTRY_ENVIRONMENT=development +``` > [!NOTE] > If using Docker Compose, defaults are set in the compose files. For production deployments, set secure secrets via your orchestrator or environment. diff --git a/docs/adding-analyzer.md b/docs/adding-analyzer.md index 0c6d0ca..224dd13 100644 --- a/docs/adding-analyzer.md +++ b/docs/adding-analyzer.md @@ -183,6 +183,8 @@ Your analyzer produces JSON in `results.json`: - The base class handles timeouts (10 minutes default) - Extracted files are automatically zipped into `.7z` archives - All exceptions are caught and logged to Sentry +- Keep analyzers idempotent and write outputs to the provided `results_dir` +- Return structured JSON so the frontend can render links/downloads automatically ## Example: Complete Analyzer From 36960f9a214d32364783b9e0d854ca35771d7e22 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Sat, 3 Jan 2026 04:03:14 +0100 Subject: [PATCH 075/102] Fix contributing --- CONTRIBUTING.md | 29 +++++++++++ README.md | 126 ++++++++++-------------------------------------- 2 files changed, 54 insertions(+), 101 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 84468c0..0af9e5b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -131,6 +131,35 @@ If your contribution needs new dependencies: 3. **Explain why** in your PR description 4. **Keep dependencies minimal** - avoid adding large libraries for small features + +## Useful commands + +```bash +# development environment (hot reload and local volumes) +docker compose -f compose.dev.yml up --build + +# Stop and remove containers, networks and volumes (results are stored in a volume) +docker compose down -v + +# Enter web container shell +docker exec -it aperisolve-web bash + +# Enter Postgres shell (from host) +docker exec -it postgres psql -U aperiuser -d aperisolve + +# Backup all uploaded files +docker cp -r aperisolve-web:/aperisolve/results /path/to/backup/location + +# Backup a single uploaded file +docker cp aperisolve-web:/aperisolve/results/filename.ext /path/to/backup/filename.ext +``` + +> [!WARNING] +> If switching between dev and production compose files, remove the `results` directory or mounted volume to avoid conflicts: +> ```bash +> rm -rf aperisolve/results +> ``` + ## Docker and Tool Installation When adding a new analyzer that requires a new tool: diff --git a/README.md b/README.md index d4be106..787d076 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,7 @@ Table of contents - [Key features](#key-features) - [Quick start (Docker)](#quick-start-docker) -- [Development](#development) -- [Configuration & environment variables](#configuration--environment-variables) - [Architecture](#architecture) -- [Troubleshooting & tips](#troubleshooting--tips) - [Roadmap](#roadmap) - [Contributing](#contributing) @@ -40,15 +37,15 @@ Table of contents - Visualize each bit layer (LSB and other layers) per image channel (R/G/B/Alpha). - Browse and download each bit-layer image. - Integrates and displays outputs from: - - [zsteg](https://github.com/zed-0xff/zsteg) (LSB text/data extraction) - - [steghide](https://steghide.sourceforge.net/) (extraction with password) - - [outguess](https://www.rbcafe.com/software/outguess/) (extraction with password) - - [openstego](https://www.openstego.com/) (extraction with password) - - [exiftool](https://exiftool.org/) (metadata and geolocation) - [binwalk](https://github.com/ReFirmLabs/binwalk) (embedded archives) + - [exiftool](https://exiftool.org/) (metadata and geolocation) - [foremost](https://foremost.sourceforge.net/) (carved files) + - [openstego](https://www.openstego.com/) (extraction with password) + - [outguess](https://www.rbcafe.com/software/outguess/) (extraction with password) - [pngcheck](https://www.libpng.org/pub/png/apps/pngcheck.html) + - [steghide](https://steghide.sourceforge.net/) (extraction with password) - [strings](https://pubs.opengroup.org/onlinepubs/9799919799/utilities/strings.html) + - [zsteg](https://github.com/zed-0xff/zsteg) (LSB text/data extraction) - Worker queue architecture for offloading heavy/slow analyzers (Redis + background workers). - Results stored for later browsing and download. @@ -56,9 +53,7 @@ Table of contents In case you want to host your own version of https://www.aperisolve.com/. -Recommended: Docker + Docker Compose. - -Clone and start the full stack (production-like): +> Required: Docker + Docker Compose. ```bash git clone https://github.com/Zeecka/AperiSolve.git @@ -67,77 +62,7 @@ cp .env.example .env docker compose up -d ``` -Default: http://localhost:5000/ - -## Development - -Development compose: - -```bash -# development environment (hot reload and local volumes) -docker compose -f compose.dev.yml up --build -``` - -Useful commands: - -```bash -# Stop and remove containers, networks and volumes (results are stored in a volume) -docker compose down -v - -# Enter web container shell -docker exec -it aperisolve-web bash - -# Enter Postgres shell (from host) -docker exec -it postgres psql -U aperiuser -d aperisolve - -# Backup all uploaded files -docker cp -r aperisolve-web:/aperisolve/results /path/to/backup/location - -# Backup a single uploaded file -docker cp aperisolve-web:/aperisolve/results/filename.ext /path/to/backup/filename.ext -``` - -> [!WARNING] -> If switching between dev and production compose files, remove the `results` directory or mounted volume to avoid conflicts: -> ```bash -> rm -rf aperisolve/results -> ``` - -## Configuration & environment variables - -The application can be configured with the `.env` file: - -```shell -# Application configuration -MAX_CONTENT_LENGTH=1048576 # Max uploaded image size (bytes) -MAX_PENDING_TIME=600 # Timeout for analyzer (seconds) -MAX_STORE_TIME=259200 # Delay until deleted from server -CLEAR_AT_RESTART=1 # Reset database and results at application restart/crash -SKIP_IHDR_FILL=0 # Skip IHDR database fill (gain times at startup but reduce results) - -# Flask configuration -FLASK_DEBUG=0 -FLASK_ENV=development - -# Redis configuration -RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0 - -# Database configuration -POSTGRES_DB=aperisolve -POSTGRES_USER=aperiuser -POSTGRES_PASSWORD=aperipass -DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve - -# Sentry configuration -SENTRY_DSN=https://your-dsn@sentry.io/project-id -SENTRY_TRACES_SAMPLE_RATE=0.1 -SENTRY_PROFILES_SAMPLE_RATE=0.1 -SENTRY_RELEASE=1.0.0 -SENTRY_ENVIRONMENT=development -``` - -> [!NOTE] -> If using Docker Compose, defaults are set in the compose files. For production deployments, set secure secrets via your orchestrator or environment. +Then browse url: http://localhost:5000/ ## Architecture @@ -148,25 +73,9 @@ SENTRY_ENVIRONMENT=development This separation keeps heavy tools (binwalk, foremost, zsteg, etc.) isolated and avoids blocking the web worker. -## Troubleshooting & tips - -- If analyzers don't produce output, check worker logs: - ```bash - docker compose logs -f aperisolve-worker - ``` -- To force re-analysis, remove results for the image (both file and in database) and re-submit the job. -- Ensure system packages needed by native tools (binwalk, foremost) are available in the analyzer containers or host image. - ## Roadmap -- [ ] **[Bug]** Duplicate key value violates unique constraint "submission_pkey" -- [ ] **[Enhancement]** Server continuous deployment -- [ ] **[Feature]** Zsteg: full extraction (--all) and download of discovered files (mp3, etc.) -- [ ] **[Feature]** Mobile-friendly UI / test on mobile -- [ ] **[Feature]** i18n (internationalization) -- [ ] **[Feature]** Rootless / unprivileged analyzers - -See [Issues](../../issues) for more! +See [Issues](../../issues). ## Contributing @@ -179,6 +88,21 @@ See [CONTRIBUTING.md](CONTRIBUTING.md). ## Credits -Acknowledgements: +Thanks to donors: +- [Philip Zimmermann](https://github.com/Philip-Zimmermann) -- Thanks to contributors and the open-source community for the tools integrated ([zsteg](https://github.com/zed-0xff/zsteg), [steghide](https://steghide.sourceforge.net/), [outguess](https://www.rbcafe.com/software/outguess/), [openstego](https://www.openstego.com/), [exiftool](https://exiftool.org/), [binwalk](https://github.com/ReFirmLabs/binwalk), [foremost](https://foremost.sourceforge.net/), [pngcheck](https://www.libpng.org/pub/png/apps/pngcheck.html), [strings](https://pubs.opengroup.org/onlinepubs/9799919799/utilities/strings.html), ...). +Thanks to contributors: +- [Zeecka](https://www.zeecka.fr/) - **(author)** +- [aradhyacp](https://github.com/aradhyacp) +- [Philip Zimmermann](https://github.com/Philip-Zimmermann) + +Thanks to the open-source community +- [binwalk](https://github.com/ReFirmLabs/binwalk) +- [exiftool](https://exiftool.org/) +- [foremost](https://foremost.sourceforge.net/) +- [openstego](https://www.openstego.com/) +- [outguess](https://www.rbcafe.com/software/outguess/) +- [pngcheck](https://www.libpng.org/pub/png/apps/pngcheck.html) +- [steghide](https://steghide.sourceforge.net/) +- [strings](https://pubs.opengroup.org/onlinepubs/9799919799/utilities/strings.html) +- [zsteg](https://github.com/zed-0xff/zsteg) From 9876034f6e0d5a3923cf6aaf4921da1a9f103af6 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Sat, 3 Jan 2026 04:10:33 +0100 Subject: [PATCH 076/102] README --- README.md | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 787d076..b065426 100644 --- a/README.md +++ b/README.md @@ -96,13 +96,5 @@ Thanks to contributors: - [aradhyacp](https://github.com/aradhyacp) - [Philip Zimmermann](https://github.com/Philip-Zimmermann) -Thanks to the open-source community -- [binwalk](https://github.com/ReFirmLabs/binwalk) -- [exiftool](https://exiftool.org/) -- [foremost](https://foremost.sourceforge.net/) -- [openstego](https://www.openstego.com/) -- [outguess](https://www.rbcafe.com/software/outguess/) -- [pngcheck](https://www.libpng.org/pub/png/apps/pngcheck.html) -- [steghide](https://steghide.sourceforge.net/) -- [strings](https://pubs.opengroup.org/onlinepubs/9799919799/utilities/strings.html) -- [zsteg](https://github.com/zed-0xff/zsteg) +Thanks to the open-source community: +[binwalk](https://github.com/ReFirmLabs/binwalk), [exiftool](https://exiftool.org/), [foremost](https://foremost.sourceforge.net/), [openstego](https://www.openstego.com/), [outguess](https://www.rbcafe.com/software/outguess/), [pngcheck](https://www.libpng.org/pub/png/apps/pngcheck.html), [steghide](https://steghide.sourceforge.net/), [strings](https://pubs.opengroup.org/onlinepubs/9799919799/utilities/strings.html), [zsteg](https://github.com/zed-0xff/zsteg), ... From aafba2f9919a2b36c31844e4a198fd06c35a452c Mon Sep 17 00:00:00 2001 From: Zeecka Date: Sat, 3 Jan 2026 04:12:31 +0100 Subject: [PATCH 077/102] Remove TOC --- README.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/README.md b/README.md index b065426..5138792 100644 --- a/README.md +++ b/README.md @@ -24,14 +24,6 @@ Support Aperi'Solve: Aperi'Solve is an open-source steganalysis web platform that performs automated analysis on images to detect and extract hidden data using common steganography tools and techniques. -Table of contents - -- [Key features](#key-features) -- [Quick start (Docker)](#quick-start-docker) -- [Architecture](#architecture) -- [Roadmap](#roadmap) -- [Contributing](#contributing) - ## Key features - Visualize each bit layer (LSB and other layers) per image channel (R/G/B/Alpha). From 68bf754762027694d99665396f8cf03e9ee25467 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sat, 3 Jan 2026 20:46:46 +0530 Subject: [PATCH 078/102] Fix viewport meta tag for improved UI responsiveness --- aperisolve/templates/base.html | 1 + 1 file changed, 1 insertion(+) diff --git a/aperisolve/templates/base.html b/aperisolve/templates/base.html index 292879c..bfd2ddc 100644 --- a/aperisolve/templates/base.html +++ b/aperisolve/templates/base.html @@ -3,6 +3,7 @@ + Aperi'Solve From 3d459e1f8bb3e170dd73df5dcc7780fb78d7af1e Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sat, 3 Jan 2026 20:53:10 +0530 Subject: [PATCH 079/102] Enhance UI responsiveness by adding box-sizing and preventing horizontal overflow --- aperisolve/static/css/aperisolve.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aperisolve/static/css/aperisolve.css b/aperisolve/static/css/aperisolve.css index 62d1901..90960a3 100644 --- a/aperisolve/static/css/aperisolve.css +++ b/aperisolve/static/css/aperisolve.css @@ -18,10 +18,15 @@ html { --bs-progress-bar-bg: #f00 !important; } +* { + box-sizing: border-box; +} + body { background-color: var(--dark-bg); color: #e0e0e0; font-family: 'Fira Code', monospace; + overflow-x: hidden; } /* ======== Typography ======== */ From 0f75666094491e849c01a2529d8e229570e89e18 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sat, 3 Jan 2026 20:54:06 +0530 Subject: [PATCH 080/102] Improve UI responsiveness by adjusting table cell dimensions and modal image sizing --- aperisolve/static/css/aperisolve.css | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/aperisolve/static/css/aperisolve.css b/aperisolve/static/css/aperisolve.css index 90960a3..9ba02b1 100644 --- a/aperisolve/static/css/aperisolve.css +++ b/aperisolve/static/css/aperisolve.css @@ -224,7 +224,6 @@ pre { background-color: var(--dark-bg-hover); box-shadow: 0 0 10px var(--theme-green); } - #upload-form { transition: height 0.4s ease, padding 0.4s ease, margin 0.4s ease; overflow: hidden; @@ -274,6 +273,7 @@ td { display: inline-block; padding: 10px; box-sizing: border-box; + min-width: 150px; } /* ======== Modal ======== */ @@ -294,7 +294,10 @@ td { } #image-modal-container img { - width: 60%; + width: 90%; + max-width: 800px; + max-height: 90vh; + object-fit: contain; border: 2px solid var(--theme-green); box-shadow: 0 0 20px var(--theme-green); cursor: pointer; @@ -309,6 +312,7 @@ td { display: inline-block; padding: 10px; width: 25%; + min-width: 150px; } /* ======== FAQ ======== */ From 499108370c32175bdde8d3684af869978b5e159c Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sat, 3 Jan 2026 20:55:10 +0530 Subject: [PATCH 081/102] Enhance UI responsiveness by adjusting image and table dimensions, and improving layout flexibility --- aperisolve/static/css/aperisolve.css | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/aperisolve/static/css/aperisolve.css b/aperisolve/static/css/aperisolve.css index 9ba02b1..e55d80f 100644 --- a/aperisolve/static/css/aperisolve.css +++ b/aperisolve/static/css/aperisolve.css @@ -354,6 +354,19 @@ td { #cheatsheet img, #cheatsheet table { margin-bottom: 15px; + max-width: 100%; + height: auto; +} + +#cheatsheet table { + min-width: auto; + width: auto; +} + +.command { + overflow-x: auto; + -webkit-overflow-scrolling: touch; + word-wrap: break-word; } /* ======== Analyzer Blocks ======== */ @@ -364,15 +377,14 @@ td { #result-infos { display: flex; + flex-wrap: wrap; + gap: 15px; } #result-infos>div { - height: 100%; - width: 50%; height: auto; - flex: 1 1 0; - width: 50%; - display: inline-block; + flex: 1 1 300px; + min-width: 280px; padding: 10px; box-sizing: border-box; vertical-align: top; From eb6e412635776a3fb29044599f22a4f54c6a5565 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sat, 3 Jan 2026 20:56:09 +0530 Subject: [PATCH 082/102] Enhance UI responsiveness with media queries for various screen sizes, adjusting image and container widths --- aperisolve/static/css/aperisolve.css | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/aperisolve/static/css/aperisolve.css b/aperisolve/static/css/aperisolve.css index e55d80f..1efc99f 100644 --- a/aperisolve/static/css/aperisolve.css +++ b/aperisolve/static/css/aperisolve.css @@ -412,6 +412,7 @@ footer { margin: 0 20px 20px 0; justify-content: flex-end !important; } + @media screen and (min-width: 1800px) { footer { position: fixed; @@ -446,3 +447,49 @@ footer a i { #httperror h1, #httperror h2{ text-align: center; } + +/* ======== Responsive Design ======== */ + +/* Large tablets and small desktops */ +@media screen and (max-width: 1200px) { + .results_img { + width: 33.333%; + } + + .browse-images { + width: 33.333%; + } +} + +/* Tablets */ +@media screen and (max-width: 992px) { + .container { + padding-left: 15px; + padding-right: 15px; + } + + .results_img { + width: 50%; + } + + .browse-images { + width: 50%; + } + + #result-infos { + flex-direction: column; + } + + #result-infos > div { + width: 100%; + flex: 1 1 100%; + } + + table { + font-size: 0.9rem; + } + + th, td { + padding: 0.5rem; + } +} \ No newline at end of file From d6ad1cf393634085002bb681d7141926843232ac Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sat, 3 Jan 2026 20:56:54 +0530 Subject: [PATCH 083/102] Enhance UI responsiveness for large phones and small tablets with media queries, adjusting font sizes, container paddings, and making tables scrollable --- aperisolve/static/css/aperisolve.css | 118 +++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/aperisolve/static/css/aperisolve.css b/aperisolve/static/css/aperisolve.css index 1efc99f..b069c3f 100644 --- a/aperisolve/static/css/aperisolve.css +++ b/aperisolve/static/css/aperisolve.css @@ -492,4 +492,122 @@ footer a i { th, td { padding: 0.5rem; } +} + + +/* Large phones and small tablets */ +@media screen and (max-width: 768px) { + body { + font-size: 14px; + } + + .container { + margin-top: 20px !important; + padding-left: 10px; + padding-right: 10px; + } + + h1 { + font-size: 1.5rem; + } + + h2 { + font-size: 1.25rem; + } + + h3 { + font-size: 1.1rem; + } + + #water-logo { + width: 50px; + } + + .card, .form-control, .table, .border, .card-hack { + border-radius: 0.35rem; + } + + .card { + padding: 15px !important; + } + + #drop-zone { + padding: 20px !important; + } + + #drop-zone p { + font-size: 0.9rem; + } + + .results_img { + width: 50%; + padding: 5px; + } + + .browse-images { + width: 50%; + padding: 5px; + } + + textarea.form-control, pre { + padding: 15px; + font-size: 0.85rem; + } + + .a-strings textarea { + height: 250px; + } + + .a-exiftool .table-container { + max-height: 300px; + } + + /* Make tables scrollable horizontally */ + .table-container { + overflow-x: auto; + -webkit-overflow-scrolling: touch; + } + + table { + font-size: 0.8rem; + min-width: 400px; + } + + th, td { + padding: 0.4rem; + word-break: break-word; + } + + footer { + justify-content: center !important; + margin: 20px 0; + flex-wrap: wrap; + gap: 15px; + } + + footer a i { + font-size: 28px; + } + + #coffee { + height: 28px; + } + + /* FAQ/Cheatsheet adjustments */ + #cheatsheet > div { + padding: 8px; + margin-bottom: 30px; + } + + #cheatsheet img { + max-width: 100%; + height: auto; + } + + .command { + padding: 8px; + font-size: 0.85rem; + overflow-x: auto; + white-space: nowrap; + } } \ No newline at end of file From 77118738df85c09d586ebf129bce28be9fe2c8f9 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sat, 3 Jan 2026 20:57:29 +0530 Subject: [PATCH 084/102] Enhance UI responsiveness for small devices by adjusting font sizes, paddings, and layout elements --- aperisolve/static/css/aperisolve.css | 116 +++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/aperisolve/static/css/aperisolve.css b/aperisolve/static/css/aperisolve.css index b069c3f..fd88544 100644 --- a/aperisolve/static/css/aperisolve.css +++ b/aperisolve/static/css/aperisolve.css @@ -610,4 +610,120 @@ footer a i { overflow-x: auto; white-space: nowrap; } +} + +/* Phones */ +@media screen and (max-width: 576px) { + body { + font-size: 13px; + } + + .container { + margin-top: 15px !important; + padding-left: 8px; + padding-right: 8px; + } + + h1 { + font-size: 1.3rem; + } + + h2 { + font-size: 1.1rem; + } + + h3 { + font-size: 1rem; + } + + #water-logo { + width: 40px; + } + + .card { + padding: 12px !important; + } + + #drop-zone { + padding: 15px !important; + } + + #drop-zone p { + font-size: 0.85rem; + } + + .btn, .btn-primary, .btn-hack { + width: 100%; + padding: 12px; + font-size: 1rem; + } + + .results_img { + width: 100%; + padding: 5px; + } + + .browse-images { + width: 100%; + padding: 5px; + } + + #result-infos > div { + padding: 5px; + } + + textarea.form-control, pre { + padding: 10px; + font-size: 0.8rem; + } + + .a-strings textarea { + height: 200px; + } + + .a-exiftool .table-container { + max-height: 250px; + } + + table { + font-size: 0.75rem; + } + + th, td { + padding: 0.3rem; + } + + .copy-icon { + display: block; + right: 5px; + margin-top: -35px; + } + + footer { + gap: 12px; + } + + footer a i { + font-size: 24px; + } + + #coffee { + height: 24px; + } + + #image-modal-container img { + width: 95%; + } + + .analyzer { + margin-top: 15px; + } + + .analyzer h2 + div { + margin-top: 10px; + } + + .analyzer h3 { + margin-top: 15px; + } } \ No newline at end of file From 77859aa9eb06ad74f8b6d01026d5d5a70a7f92fd Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sat, 3 Jan 2026 20:57:52 +0530 Subject: [PATCH 085/102] Enhance FAQ/Cheatsheet layout by adjusting padding and font sizes for list items and commands --- aperisolve/static/css/aperisolve.css | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/aperisolve/static/css/aperisolve.css b/aperisolve/static/css/aperisolve.css index fd88544..70cafff 100644 --- a/aperisolve/static/css/aperisolve.css +++ b/aperisolve/static/css/aperisolve.css @@ -726,4 +726,19 @@ footer a i { .analyzer h3 { margin-top: 15px; } + + /* FAQ/Cheatsheet adjustments */ + #cheatsheet li { + padding-left: 20px; + font-size: 0.9rem; + } + + .command { + font-size: 0.75rem; + padding: 6px; + } + + #httperror { + padding-top: 50px; + } } \ No newline at end of file From f60925be9b97e2ee23ce78a9a6097e695ae53383 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sat, 3 Jan 2026 20:58:43 +0530 Subject: [PATCH 086/102] Enhance UI responsiveness for extra small devices and touch optimizations with media queries --- aperisolve/static/css/aperisolve.css | 94 +++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 2 deletions(-) diff --git a/aperisolve/static/css/aperisolve.css b/aperisolve/static/css/aperisolve.css index 70cafff..732a323 100644 --- a/aperisolve/static/css/aperisolve.css +++ b/aperisolve/static/css/aperisolve.css @@ -726,7 +726,7 @@ footer a i { .analyzer h3 { margin-top: 15px; } - + /* FAQ/Cheatsheet adjustments */ #cheatsheet li { padding-left: 20px; @@ -741,4 +741,94 @@ footer a i { #httperror { padding-top: 50px; } -} \ No newline at end of file +} + +/* Extra small phones */ +@media screen and (max-width: 380px) { + body { + font-size: 12px; + } + + h1 { + font-size: 1.1rem; + } + + #water-logo { + width: 35px; + } + + .btn, .btn-primary, .btn-hack { + padding: 10px; + font-size: 0.9rem; + } + + footer a i { + font-size: 22px; + } + + #coffee { + height: 22px; + } +} + + +/* Touch device optimizations */ +@media (hover: none) and (pointer: coarse) { + .copy-icon { + display: block; + opacity: 0.7; + } + + .btn, .btn-primary, .btn-hack { + min-height: 44px; + } + + input, textarea, select { + font-size: 16px; /* Prevents zoom on iOS */ + } + + a, button { + min-height: 44px; + min-width: 44px; + } +} + +/* Landscape mode on phones */ +@media screen and (max-height: 500px) and (orientation: landscape) { + .container { + margin-top: 10px !important; + } + + #image-modal-container img { + max-height: 85vh; + width: auto; + max-width: 95%; + } +} + +/* Print styles */ +@media print { + #three-bg { + display: none; + } + + body { + background-color: white; + color: black; + } + + .card, .form-control, .table, .border, .card-hack { + background-color: white; + border-color: #333 !important; + box-shadow: none; + } + + h1, h2, h3, h4, h5, h6, strong, .highlight { + color: #333; + text-shadow: none; + } + + a, .text-accent { + color: #333; + } +} From f5a4bb99e4cd0679100c76d6f2a6a342a8daf9aa Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sat, 3 Jan 2026 23:54:52 +0530 Subject: [PATCH 087/102] Add SKIP_IHDR_FILL environment variable for worker service --- compose.dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.dev.yml b/compose.dev.yml index 77baee1..afb1244 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -39,6 +39,7 @@ services: - FLASK_DEBUG=true - FLASK_ENV=development - CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing + - SKIP_IHDR_FILL=1 redis: image: redis:7 From cc0da503537288569623bfed1916e2fac864eea9 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sat, 3 Jan 2026 23:55:00 +0530 Subject: [PATCH 088/102] Add SKIP_IHDR_FILL environment variable to worker service --- compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.yml b/compose.yml index 4489e66..07c2c59 100644 --- a/compose.yml +++ b/compose.yml @@ -39,6 +39,7 @@ services: - .env environment: - CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing + - SKIP_IHDR_FILL=1 postgres: image: postgres:18 From 251902d22c57d2649e70d7047317b0aec5ef650b Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sat, 3 Jan 2026 23:55:07 +0530 Subject: [PATCH 089/102] Skip IHDR table population and database initialization when running as a worker --- aperisolve/models.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/aperisolve/models.py b/aperisolve/models.py index 22d6d34..300f285 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -8,6 +8,7 @@ from datetime import datetime, timezone from os import getenv from pathlib import Path from shutil import rmtree +import sys from flask import Flask from flask_sqlalchemy import SQLAlchemy @@ -73,6 +74,11 @@ def fill_ihdr_db() -> None: Returns: None """ + # Skip if table already has entries + if db.session.query(IHDR.iid).first() is not None: + print("IHDR table already populated, skipping fill.") + return + # Standard PNG IHDR parameters resolutions = get_resolutions() bit_color_pairs = get_valid_depth_color_pairs() @@ -127,6 +133,12 @@ def init_db(app: Flask) -> None: - Populates the IHDR CRC lookup table with initial data - Prints status messages to console during initialization """ + # Workers should never clear/fill the database + # Detect if running as RQ worker + if 'rq.worker' in sys.modules: + print("Running as worker, skipping database initialization.") + return + with app.app_context(): if getenv("CLEAR_AT_RESTART", "0") == "1": # Force clear if CLEAR_AT_RESTART print("Clearing database and file system at restart...") From f0c77f8a5d717b722b449e6e447aaf7d19310c70 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sun, 4 Jan 2026 00:05:12 +0530 Subject: [PATCH 090/102] Remove database initialization check for RQ worker in init_db function --- aperisolve/models.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/aperisolve/models.py b/aperisolve/models.py index 300f285..72bc8f1 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -8,7 +8,6 @@ from datetime import datetime, timezone from os import getenv from pathlib import Path from shutil import rmtree -import sys from flask import Flask from flask_sqlalchemy import SQLAlchemy @@ -133,12 +132,6 @@ def init_db(app: Flask) -> None: - Populates the IHDR CRC lookup table with initial data - Prints status messages to console during initialization """ - # Workers should never clear/fill the database - # Detect if running as RQ worker - if 'rq.worker' in sys.modules: - print("Running as worker, skipping database initialization.") - return - with app.app_context(): if getenv("CLEAR_AT_RESTART", "0") == "1": # Force clear if CLEAR_AT_RESTART print("Clearing database and file system at restart...") From 93564d2e5fe59a84b4369f8b6f34b199c64ad031 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sun, 4 Jan 2026 00:05:21 +0530 Subject: [PATCH 091/102] Add CLEAR_AT_RESTART environment variable to web service for Flask debug --- compose.dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.dev.yml b/compose.dev.yml index afb1244..19f3af9 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -21,6 +21,7 @@ services: environment: - FLASK_DEBUG=true - FLASK_ENV=development + - CLEAR_AT_RESTART=0 # Don't clear on Flask debug auto-reload worker: #image: ghcr.io/zeecka/aperisolve:latest From 2ce2b82b494333fcbdf4dd65984df2c9ba6e3766 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sun, 4 Jan 2026 00:15:00 +0530 Subject: [PATCH 092/102] Enhance IHDR table fill logging for clarity --- aperisolve/models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/aperisolve/models.py b/aperisolve/models.py index 72bc8f1..5ae1436 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -110,7 +110,7 @@ def fill_ihdr_db() -> None: db.session.commit() db.session.commit() # pylint: disable=no-member - print(f"Precomputed {count} common IHDR entries.") + print(f"Precomputed {count} common IHDR entries. IHDR table filled successfully.") def init_db(app: Flask) -> None: @@ -145,6 +145,4 @@ def init_db(app: Flask) -> None: if getenv("SKIP_IHDR_FILL", "0") == "1": print("Skipping IHDR lookup table fill as per configuration.") else: - print("Filling IHDR lookup table...") fill_ihdr_db() - print("IHDR table filled successfully.") From a7783e174e40e2a1168471c9a1b2d87c8d488f79 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sun, 4 Jan 2026 00:15:35 +0530 Subject: [PATCH 093/102] Add logging for IHDR lookup table fill process --- aperisolve/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aperisolve/models.py b/aperisolve/models.py index 5ae1436..157459b 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -145,4 +145,5 @@ def init_db(app: Flask) -> None: if getenv("SKIP_IHDR_FILL", "0") == "1": print("Skipping IHDR lookup table fill as per configuration.") else: + print("Filling IHDR lookup table...") fill_ihdr_db() From f5dd083d10f9bf8396a8b85ecd4e19e998faf771 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sun, 4 Jan 2026 01:26:57 +0530 Subject: [PATCH 094/102] Remove SKIP_IHDR_FILL environment variable from worker service --- compose.dev.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/compose.dev.yml b/compose.dev.yml index 19f3af9..ce677ea 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -40,7 +40,6 @@ services: - FLASK_DEBUG=true - FLASK_ENV=development - CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing - - SKIP_IHDR_FILL=1 redis: image: redis:7 From 370405ab24088a3ae1331867b1036f1092c37906 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sun, 4 Jan 2026 01:27:05 +0530 Subject: [PATCH 095/102] Remove SKIP_IHDR_FILL environment variable from worker service --- compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/compose.yml b/compose.yml index 07c2c59..4489e66 100644 --- a/compose.yml +++ b/compose.yml @@ -39,7 +39,6 @@ services: - .env environment: - CLEAR_AT_RESTART=0 # Always false for worker to prevent double clearing - - SKIP_IHDR_FILL=1 postgres: image: postgres:18 From 4faf7fd9b517e6694fb9fd6ac2587eb4da3fc0c6 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sun, 4 Jan 2026 01:27:12 +0530 Subject: [PATCH 096/102] Skip database initialization when running as RQ worker --- aperisolve/models.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/aperisolve/models.py b/aperisolve/models.py index 157459b..cda1c9e 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -4,6 +4,7 @@ """This module defines the database models for the Aperi'Solve application.""" import itertools +import sys from datetime import datetime, timezone from os import getenv from pathlib import Path @@ -132,6 +133,13 @@ def init_db(app: Flask) -> None: - Populates the IHDR CRC lookup table with initial data - Prints status messages to console during initialization """ + # Detect if running as RQ worker (not wsgi/web) + is_worker = len(sys.argv) > 0 and ("rq" in sys.argv[0] or "rq.worker" in sys.modules) + + if is_worker: + print("Running as worker, skipping database initialization.") + return + with app.app_context(): if getenv("CLEAR_AT_RESTART", "0") == "1": # Force clear if CLEAR_AT_RESTART print("Clearing database and file system at restart...") @@ -142,8 +150,6 @@ def init_db(app: Flask) -> None: db.create_all() print("Database structure created successfully.") - if getenv("SKIP_IHDR_FILL", "0") == "1": - print("Skipping IHDR lookup table fill as per configuration.") - else: - print("Filling IHDR lookup table...") - fill_ihdr_db() + print("Filling IHDR lookup table...") + fill_ihdr_db() + From 2e54cf0b468899122499998b2718673ae1dc5718 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sun, 4 Jan 2026 01:27:57 +0530 Subject: [PATCH 097/102] Refactor worker detection logic in database initialization --- aperisolve/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aperisolve/models.py b/aperisolve/models.py index cda1c9e..358ed5b 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -134,7 +134,8 @@ def init_db(app: Flask) -> None: - Prints status messages to console during initialization """ # Detect if running as RQ worker (not wsgi/web) - is_worker = len(sys.argv) > 0 and ("rq" in sys.argv[0] or "rq.worker" in sys.modules) + # rq.worker module is loaded when RQ runs a worker process + is_worker = "rq.worker" in sys.modules if is_worker: print("Running as worker, skipping database initialization.") From 6bdcf990452890577149036de8d4de0b8034e412 Mon Sep 17 00:00:00 2001 From: aradhyacp Date: Sun, 4 Jan 2026 01:35:27 +0530 Subject: [PATCH 098/102] Improve database initialization logic for RQ workers --- aperisolve/models.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/aperisolve/models.py b/aperisolve/models.py index 358ed5b..e9f82b6 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -133,24 +133,26 @@ def init_db(app: Flask) -> None: - Populates the IHDR CRC lookup table with initial data - Prints status messages to console during initialization """ - # Detect if running as RQ worker (not wsgi/web) - # rq.worker module is loaded when RQ runs a worker process - is_worker = "rq.worker" in sys.modules - - if is_worker: - print("Running as worker, skipping database initialization.") - return + # Detect if running as RQ worker by checking command line + # Worker: /usr/local/bin/rq worker ... + # Web: /usr/local/bin/flask run ... or gunicorn/wsgi + is_worker = any("rq" in arg and "flask" not in arg for arg in sys.argv[:2]) with app.app_context(): - if getenv("CLEAR_AT_RESTART", "0") == "1": # Force clear if CLEAR_AT_RESTART + # Workers should never clear the database + if not is_worker and getenv("CLEAR_AT_RESTART", "0") == "1": print("Clearing database and file system at restart...") db.session.remove() # pylint: disable=no-member db.drop_all() rmtree(Path("./results"), ignore_errors=True) # Clear results folder + # Always create tables (idempotent - safe to call multiple times) db.create_all() print("Database structure created successfully.") - print("Filling IHDR lookup table...") - fill_ihdr_db() + # Workers should never fill IHDR table + if is_worker: + print("Running as worker, skipping IHDR table fill.") + else: + fill_ihdr_db() From 8176d0852ac59208c3839327e794a0e5f26661e6 Mon Sep 17 00:00:00 2001 From: Zeecka <24873615+Zeecka@users.noreply.github.com> Date: Sun, 4 Jan 2026 04:59:14 +0100 Subject: [PATCH 099/102] Update CONTRIBUTING.md - remove email placeholder --- CONTRIBUTING.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0af9e5b..049d550 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -196,7 +196,6 @@ RUN git clone https://github.com/tool/repo.git /tmp/tool && \ - 💬 **Questions?** Open a [Discussion](../../discussions) - 🐛 **Found a bug?** Open an [Issue](../../issues) -- 📧 **Email:** [Your contact email] - 💡 **Ideas?** We'd love to hear them in Discussions! ## Recognition From f26c638c5233fb2677613fbb967c2d8e0d2ed64e Mon Sep 17 00:00:00 2001 From: Zeecka <24873615+Zeecka@users.noreply.github.com> Date: Sun, 4 Jan 2026 05:00:52 +0100 Subject: [PATCH 100/102] Update CONTRIBUTING.md - remove license related paragraph --- CONTRIBUTING.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 049d550..88ab0eb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to Aperi'Solve +u# Contributing to Aperi'Solve Thank you for your interest in contributing! We welcome all contributions, whether it's bug fixes, new features, or documentation improvements. @@ -203,10 +203,6 @@ RUN git clone https://github.com/tool/repo.git /tmp/tool && \ > [!NOTE] > All contributors will be recognized in our [Contributors](../../graphs/contributors) page and in release notes! -## License - -By contributing, you agree that your contributions will be licensed under the same license as the project (see [LICENSE](LICENSE)). - --- **Thank you for contributing! 🎉** From ab5e3e26fe49727633eb95c2782387bead488a51 Mon Sep 17 00:00:00 2001 From: Zeecka Date: Sun, 4 Jan 2026 18:19:54 +0100 Subject: [PATCH 101/102] CD --- .github/workflows/deployement.yml | 61 +++++++++++++++++++++++++++++++ .github/workflows/releases.yml | 25 +++++++++---- 2 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/deployement.yml diff --git a/.github/workflows/deployement.yml b/.github/workflows/deployement.yml new file mode 100644 index 0000000..2cdfaf3 --- /dev/null +++ b/.github/workflows/deployement.yml @@ -0,0 +1,61 @@ +name: Deploy to production + +on: + workflow_run: + workflows: + - Docker Image CI + types: + - completed + +jobs: + deploy: + # ne déploie que si le build a réussi + if: ${{ github.event.workflow_run.conclusion == 'success' }} + + runs-on: ubuntu-latest + environment: production + + steps: + - name: Deploy to production server + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ secrets.PROD_SSH_HOST }} + username: ${{ secrets.PROD_SSH_USER }} + key: ${{ secrets.PROD_SSH_KEY }} + script: | + set -e + cd /root/AperiSolve + + # update code + git pull + + # create .env + cat < .env + MAX_CONTENT_LENGTH=${{ vars.MAX_CONTENT_LENGTH }} + MAX_PENDING_TIME=${{ vars.MAX_PENDING_TIME }} + MAX_STORE_TIME=${{ vars.MAX_STORE_TIME }} + CLEAR_AT_RESTART=${{ vars.CLEAR_AT_RESTART }} + SKIP_IHDR_FILL=${{ vars.SKIP_IHDR_FILL }} + + FLASK_DEBUG=${{ vars.FLASK_DEBUG }} + FLASK_ENV=${{ vars.FLASK_ENV }} + + RQ_DASHBOARD_REDIS_URL=${{ secrets.REDIS_URL }} + + POSTGRES_USER=${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} + POSTGRES_DB=${{ secrets.POSTGRES_DB }} + DB_URI=${{ secrets.DATABASE_URL }} + + SENTRY_DSN=${{ secrets.SENTRY_DSN }} + SENTRY_TRACES_SAMPLE_RATE=${{ secrets.SENTRY_TRACES_SAMPLE_RATE }} + SENTRY_PROFILES_SAMPLE_RATE=${{ secrets.SENTRY_PROFILES_SAMPLE_RATE }} + SENTRY_RELEASE=${{ secrets.SENTRY_RELEASE }} + SENTRY_ENVIRONMENT=${{ secrets.SENTRY_ENVIRONMENT }} + EOF + + # Pull images and deploy + docker compose pull + docker compose down + docker compose up -d + docker image prune -f diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 6d780f8..561cabc 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -10,22 +10,24 @@ jobs: docker-releases: runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') && (github.ref_name == 'main' || github.ref_name == 'prod') + steps: - name: Get release version run: echo "RELEASE_VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - + - name: downcase username run: echo "lower_actor=${GITHUB_ACTOR,,}" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v4 - + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -33,12 +35,21 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Determine latest tag + run: | + if [[ "${GITHUB_REF_NAME}" == "main" ]]; then + echo "LATEST_TAG=latest" >> $GITHUB_ENV + elif [[ "${GITHUB_REF_NAME}" == "prod" ]]; then + echo "LATEST_TAG=latest-prod" >> $GITHUB_ENV + else + echo "LATEST_TAG=" >> $GITHUB_ENV + fi + - name: Build and push Docker images uses: docker/build-push-action@v6.9.0 with: context: ./ push: true tags: | - ghcr.io/${{ env.lower_actor }}/aperisolve:latest - ghcr.io/${{ env.lower_actor }}/aperisolve:${{ env.RELEASE_VERSION }} - \ No newline at end of file + ghcr.io/${{ env.lower_actor }}/aperisolve:${{ env.RELEASE_VERSION }} + ghcr.io/${{ env.lower_actor }}/aperisolve:${{ env.LATEST_TAG }} From aef8d89f4f97dc93ab3e022772a9276f70eb4a8c Mon Sep 17 00:00:00 2001 From: Zeecka Date: Sun, 4 Jan 2026 18:22:13 +0100 Subject: [PATCH 102/102] lint and gitignore --- .gitignore | 1 + aperisolve/models.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8a70dc2..7b0fdd9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ __pycache__/ .mypy_cache/ results/ +.vscode diff --git a/aperisolve/models.py b/aperisolve/models.py index e9f82b6..3f13c4d 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -78,7 +78,7 @@ def fill_ihdr_db() -> None: if db.session.query(IHDR.iid).first() is not None: print("IHDR table already populated, skipping fill.") return - + # Standard PNG IHDR parameters resolutions = get_resolutions() bit_color_pairs = get_valid_depth_color_pairs() @@ -155,4 +155,3 @@ def init_db(app: Flask) -> None: print("Running as worker, skipping IHDR table fill.") else: fill_ihdr_db() -