diff --git a/.env b/.env index bdea06b..7d7e694 100644 --- a/.env +++ b/.env @@ -7,4 +7,4 @@ DB_URI=postgresql://aperiuser:aperipass@postgres:5432/aperisolve RQ_DASHBOARD_REDIS_URL=redis://redis:6379/0 POSTGRES_DB=aperisolve POSTGRES_USER=aperiuser -POSTGRES_PASSWORD=aperipass \ No newline at end of file +POSTGRES_PASSWORD=aperipass diff --git a/.gitignore b/.gitignore index aa5b26a..646de31 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ # Python __pycache__/ .mypy_cache/ -results/ \ No newline at end of file +results/ diff --git a/Dockerfile b/Dockerfile index 67235ca..5e4eb5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,10 @@ RUN apt-get update && apt-get install -y \ COPY aperisolve/ /aperisolve/ +RUN mkdir -p /data + RUN pip install --no-cache-dir -r /aperisolve/requirements.txt -CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "aperisolve.wsgi:application"] \ No newline at end of file +ENV PYTHONUNBUFFERED=1 + +CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "--access-logfile", "-", "--error-logfile", "-", "--log-level", "info", "--capture-output", "aperisolve.wsgi:application"] \ No newline at end of file diff --git a/aperisolve/analyzers/foremost.py b/aperisolve/analyzers/foremost.py index 7e82452..5c55972 100644 --- a/aperisolve/analyzers/foremost.py +++ b/aperisolve/analyzers/foremost.py @@ -24,7 +24,7 @@ def analyze_foremost(input_img: Path, output_dir: Path) -> None: timeout=MAX_PENDING_TIME, ) # Note: foremost use stderr as standard output for results :o) - # So we can't use it as error dectection : + # So we can't use it as error detection : # stderr += data.stderr # Zip extracted files diff --git a/aperisolve/analyzers/image_resize.py b/aperisolve/analyzers/image_resize.py index c9ed704..7fdfcb4 100644 --- a/aperisolve/analyzers/image_resize.py +++ b/aperisolve/analyzers/image_resize.py @@ -1,165 +1,118 @@ """Image size bruteforcer for common PNG sizes.""" -import zlib from pathlib import Path -from .utils import update_data +from typing import List -# Full list of expected sizes -EXPECTED_SIZES = [ - # ------------------------- - # ICONS (Square) - # ------------------------- - (16, 16), - (20, 20), - (24, 24), - (28, 28), - (32, 32), - (48, 48), - (64, 64), - (72, 72), - (96, 96), - (128, 128), - (192, 192), - (256, 256), - (384, 384), - (512, 512), - (768, 768), - (1024, 1024), - (1536, 1536), - (2048, 2048), - (4096, 4096), - # ------------------------- - # 1:1 Square (General) - # ------------------------- - (300, 300), - (500, 500), - (800, 800), - (1000, 1000), - (1500, 1500), - (2000, 2000), - (2500, 2500), - (3000, 3000), - # ------------------------- - # 4:3 Aspect (Classic Screens) - # ------------------------- - (320, 240), - (640, 480), - (800, 600), - (960, 720), - (1024, 768), - (1280, 960), - (1600, 1200), - (2048, 1536), - (2560, 1920), - (3200, 2400), - (4096, 3072), - # ------------------------- - # 3:2 Aspect (Photography) - # ------------------------- - (600, 400), - (900, 600), - (1200, 800), - (1500, 1000), - (1800, 1200), - (2400, 1600), - (3000, 2000), - (3600, 2400), - (4500, 3000), - (6000, 4000), - # ------------------------- - # 16:9 Aspect (Video / Web) - # ------------------------- - (426, 240), - (640, 360), - (854, 480), - (960, 540), - (1280, 720), - (1366, 768), - (1600, 900), - (1920, 1080), - (2560, 1440), - (3200, 1800), - (3440, 1935), - (3840, 2160), - (5120, 2880), - (7680, 4320), - (10240, 5760), - # ------------------------- - # 16:10 Aspect (Laptops) - # ------------------------- - (1280, 800), - (1440, 900), - (1680, 1050), - (1920, 1200), - (2560, 1600), - (2880, 1800), - (3840, 2400), - (5120, 3200), - # ------------------------- - # Mobile Portrait (9:16) - # ------------------------- - (720, 1280), - (1080, 1920), - (1440, 2560), - (2160, 3840), - (1440, 2960), - (1080, 2400), - (1170, 2532), - (1284, 2778), - (1290, 2796), - # ------------------------- - # Social Media Sizes - # ------------------------- - # Instagram - (1080, 1080), - (1080, 1350), - (1080, 1920), - # Facebook - (1200, 628), - (1200, 1200), - # Twitter/X - (1200, 675), - (1500, 500), - # YouTube Thumbnails - (1280, 720), - (1920, 1080), - # ------------------------- - # Banners / Headers - # ------------------------- - (1600, 500), - (1920, 500), - (2560, 700), - (3000, 1000), - (3840, 1200), - # ------------------------- - # Print-like Digital Sizes - # ------------------------- - (2550, 3300), # Letter @ 300 DPI - (2480, 3508), # A4 @ 300 DPI - (3508, 4961), # A3 @ 300 DPI - (4961, 7016), # A2 @ 300 DPI - (7016, 9933), # A1 @ 300 DPI -] +from ..app import app +from ..models import IHDR +from .utils import PNG, update_data -def calc_checksum( - header_chunk: bytearray, width_bytes: bytearray, height_bytes: bytearray -) -> bytearray: +def write_recovered_png( + png_bytes: bytearray, + width: int, + height: int, + output_path: Path, +) -> None: """ - Calculates the CRC32 of the IHDR chunk with new dimensions. - header_chunk: The full IHDR chunk (Type + Data). + Rebuilds a PNG by patching IHDR width/height and writes it to disk. """ - # Reconstruct IHDR: [Type(4)] + [New Width(4)] + [New Height(4)] + [Rest of Data(5)] - new_header = header_chunk[:4] + width_bytes + height_bytes + header_chunk[12:] + height_bytes = height.to_bytes(4, byteorder="big") + width_bytes = width.to_bytes(4, byteorder="big") - # Calculate CRC and return as 4 bytes (Big Endian) - return bytearray((zlib.crc32(new_header) & 0xFFFFFFFF).to_bytes(4, byteorder="big")) + full_png_data = ( + png_bytes[:16] # Splicing: [Start...IHDR+4] + [W] + [H] + [IHDR+12...End] + + width_bytes + + height_bytes + + png_bytes[24:] + ) + + with output_path.open("wb") as out_f: + out_f.write(full_png_data) + + +def lookup_crc(png: PNG) -> List[PNG]: + """ + Look up PNG images in the database by their CRC value. + + Args: + png (PNG): A PNG object containing the CRC value to search for. + + Returns: + List[PNG]: A list of PNG objects with matching CRC values, reconstructed from their packed + representations. + """ + with app.app_context(): + imgs = IHDR.query.filter_by(crc=png.crc).all() + results = [] + for img in imgs: + packed = img.packed + results.append(PNG.from_packed(packed, crc=png.crc)) + return results + + +def search_height_crc(png: PNG) -> List[PNG]: + """ + Search for PNG images with matching CRC by iterating through possible heights. + + This function attempts to find alternative PNG configurations that produce the same + CRC checksum as the input PNG by systematically varying the height parameter while + keeping all other properties constant. + + Args: + png (PNG): The reference PNG object whose CRC checksum will be matched. + + Returns: + List[PNG]: A list of PNG objects with matching CRC values. Returns an empty list + if no matches are found. + + Note: + The function iterates through heights from 100 to 3500 (inclusive). + This may be computationally expensive for large ranges. + """ + results = [] + for height in range(100, 3501): + candidate = PNG( + width=png.width, + height=height, + bit_depth=png.bit_depth, + color_type=png.color_type, + interlace=png.interlace, + ) + if candidate.crc == png.crc: + results.append(candidate) + return results def analyze_image_resize(input_img: Path, output_dir: Path) -> None: """ - Analyze an image submission using python-based resize bruteforce. - Attempts to recover PNGs with modified resolutions but valid CRCs. + Analyze a PNG image and recover resized versions based on CRC matching. + + This function attempts to find matching image dimensions for a given PNG file + by looking up its CRC (Cyclic Redundancy Check) in a database. If matches are + found, it recovers and saves PNG images with the matched dimensions. + + Args: + input_img (Path): Path to the input PNG image file to analyze. + output_dir (Path): Path to the directory where recovered PNG images will be saved. + + Returns: + None + + Raises: + Handles exceptions gracefully by updating output data with error status. + + Side Effects: + - Creates output directory if it doesn't exist. + - Saves recovered PNG images to the output directory. + - Updates data in the output directory with analysis results via update_data(). + + Notes: + - If the PNG has invalid structure or IHDR chunk is not first, an error is logged. + - The function performs a two-step lookup: first by CRC, then by height/CRC if no matches + found. + - Recovered images are named as "recovered_{width}x{height}.png". + - Analysis results (logs, status, and image URLs) are persisted via update_data(). """ # 1. Setup Paths @@ -168,81 +121,61 @@ def analyze_image_resize(input_img: Path, output_dir: Path) -> None: # 3. Initialize Logs logs = [] - recovered_image = None try: with input_img.open("rb") as image: - f = image.read() + img_bytes = bytearray(image.read()) + png = PNG.from_bytearray(img_bytes) - b = bytearray(f) - - # Find IHDR start - ihdr = b.find(b"\x49\x48\x44\x52") - if ihdr == -1: - logs.append("Failure: PNG header (IHDR) not found.") + if png is None: + logs.append( + "Failure: IHDR chunk is not the first chunk, or PNG has invalid structure." + ) update_data( output_dir, - {"image_resize": {"status": "error", "error": "PNG header not found."}}, + { + "image_resize": { + "status": "error", + "error": "IHDR chunk is not the first chunk, or PNG has invalid structure.", + } + }, ) return - # Calculate Chunk Length (The 4 bytes BEFORE IHDR tag) - chunk_length = int.from_bytes((b[ihdr - 4 : ihdr]), byteorder="big") + 4 + logs.append(f"Target CRC found: 0x{png.crc:08x}") - # Extract Target CRC (The 4 bytes AFTER the chunk data) - target_crc = b[ihdr + chunk_length : ihdr + chunk_length + 4] + saved_img_urls = [] - # Isolate the header chunk (Type + Data) - header_chunk = b[ihdr : ihdr + chunk_length] + db_matches = lookup_crc(png) + if not db_matches: + db_matches = search_height_crc(png) + if not db_matches: + logs.append("Failure: No matching dimensions found.") + else: + # Create parent directories if they don't exist + output_dir.mkdir(parents=True, exist_ok=True) - logs.append(f"Target CRC found: 0x{target_crc.hex()}") - - match_found = False - - # Brute-force Loop - for size in EXPECTED_SIZES: - width, height = size - - # Convert dimensions to 4-byte Big Endian arrays - width_bytes = bytearray(width.to_bytes(4, byteorder="big")) - height_bytes = bytearray(height.to_bytes(4, byteorder="big")) - - # Check if this size matches the file's CRC - if target_crc == calc_checksum(header_chunk, width_bytes, height_bytes): - match_found = True - logs.append(f"Success: Match found! Dimensions: {width}x{height}") - - # Create Output Filename - img_name = f"recovered_{width}x{height}.png" - - # Create parent directories if they don't exist - output_dir.mkdir(parents=True, exist_ok=True) + # Iterate through ALL candidates found + for im_match in db_matches: + img_name = f"recovered_{im_match.width}x{im_match.height}.png" output_path = output_dir / img_name - - # Splicing: [Start...IHDR+4] + [W] + [H] + [IHDR+12...End] - full_png_data = ( - b[: ihdr + 4] + width_bytes + height_bytes + b[ihdr + 12 :] + write_recovered_png( + img_bytes, im_match.width, im_match.height, output_path ) - # Write the recovered file to disk - with output_path.open("wb") as out_f: - out_f.write(full_png_data) - - recovered_image = Path(output_dir.name) / img_name - break # Stop looking after finding the match - - if not match_found: - logs.append("Failure: No matching dimensions found.") + logs.append(f"Image saved: {img_name}") + # Append the formatted URL to our list + saved_img_urls.append("/image/" + str(Path(output_dir.name) / img_name)) # 4. Final Data Update output_data = { - "image_resize": { # Key name must match your module name in AperiSolve + "image_resize": { "status": "ok", "output": logs, + "png_images": saved_img_urls, } } - if recovered_image: - output_data["image_resize"]["image"] = "/image/" + str(recovered_image) + update_data(output_dir, output_data) except Exception as e: diff --git a/aperisolve/analyzers/outguess.py b/aperisolve/analyzers/outguess.py index 1d6587e..976979b 100644 --- a/aperisolve/analyzers/outguess.py +++ b/aperisolve/analyzers/outguess.py @@ -11,7 +11,7 @@ from .utils import MAX_PENDING_TIME, update_data def analyze_outguess( input_img: Path, output_dir: Path, password: Optional[str] = None ) -> None: - """Analyze an image submission using stehide.""" + """Analyze an image submission using outguess.""" image_name = input_img.name try: diff --git a/aperisolve/analyzers/pngcheck.py b/aperisolve/analyzers/pngcheck.py index fa32faf..5c9a945 100644 --- a/aperisolve/analyzers/pngcheck.py +++ b/aperisolve/analyzers/pngcheck.py @@ -7,7 +7,7 @@ from .utils import MAX_PENDING_TIME, update_data def analyze_pngcheck(input_img: Path, output_dir: Path) -> None: - """Analyze an image submission using strings.""" + """Analyze an image submission using pngcheck.""" try: data = subprocess.run( @@ -24,15 +24,15 @@ def analyze_pngcheck(input_img: Path, output_dir: Path) -> None: # ) # return - data_strings = data.stdout.split("\n") if data else [] - data_strings = [s for s in data_strings if s] + data_pngcheck = data.stdout.split("\n") if data else [] + data_pngcheck = [s for s in data_pngcheck if s] update_data( output_dir, { "pngcheck": { "status": "ok", - "output": data_strings, + "output": data_pngcheck, } }, ) diff --git a/aperisolve/analyzers/steghide.py b/aperisolve/analyzers/steghide.py index 8a12ce9..d74d077 100644 --- a/aperisolve/analyzers/steghide.py +++ b/aperisolve/analyzers/steghide.py @@ -12,7 +12,7 @@ from .utils import MAX_PENDING_TIME, update_data def analyze_steghide( input_img: Path, output_dir: Path, password: Optional[str] = None ) -> None: - """Analyze an image submission using stehide.""" + """Analyze an image submission using steghide.""" image_name = input_img.name try: diff --git a/aperisolve/analyzers/utils.py b/aperisolve/analyzers/utils.py index a20036a..df7da95 100644 --- a/aperisolve/analyzers/utils.py +++ b/aperisolve/analyzers/utils.py @@ -2,10 +2,14 @@ import fcntl import json +import math import os +import struct import threading +import zlib +from functools import cached_property from pathlib import Path -from typing import Any +from typing import Any, Self _thread_lock = threading.Lock() @@ -46,3 +50,255 @@ def update_data( os.replace(tmp_file, json_file) # ensures file write is atomic finally: fcntl.flock(lock, fcntl.LOCK_UN) + + +def get_resolutions(): + """ + Generate a sorted list of common display and image resolutions. + + Creates a comprehensive set of resolution tuples by combining various base widths + with standard aspect ratios used in screens, digital displays, photography, and + paper standards. Includes both portrait and landscape orientations. + + Base widths range from 16px to 10000px, covering: + - Low resolution: 16-256px (16px increments) + - Standard: 320-1024px (32px increments) + - HD/Full HD: 1280-2560px (64px increments) + - 4K: 3000-4096px (128px increments) + - 5K-8K: 5120-8192px (256px increments) + - Upper bound: 10000px + + Aspect ratios include common formats: + - Digital screens: 1:1, 4:3, 3:2, 16:10, 16:9, 21:9 + - Photography/print: 5:4, 7:5, 2:3 + - Paper standards: ISO A-series, US Letter, Legal, Tabloid + + Returns: + list[tuple[int, int]]: Sorted list of (width, height) resolution tuples + with heights constrained between 1 and 10000 pixels. + """ + base_widths = ( + list(range(16, 257, 16)) # 16 → 256 + + list(range(320, 1025, 32)) # 320 → 1024 + + list(range(1280, 2561, 64)) # 1280 → 2560 + + list(range(3000, 4097, 128)) # 3K–4K + + list(range(5120, 8193, 256)) # 5K–8K + + [10000] # upper bound + ) + aspect_ratios = [ + # Screens / digital + (1, 1), # square + (4, 3), + (3, 2), + (16, 10), + (16, 9), + (21, 9), + # Photography / print + (5, 4), # 8x10 + (7, 5), # 5x7 + (3, 2), # 4x6 (already included, kept for clarity) + (2, 3), # poster + # Paper standards + (1414, 1000), # ISO A-series (A4, A3, A2, ...) + (11, 85), # US Letter + (14, 85), # Legal + (17, 11), # Tabloid + ] + resolutions = set() + for w in base_widths: + for ar_w, ar_h in aspect_ratios: + h = math.floor(round(w * ar_h / ar_w)) + if 1 <= h <= 10000: + resolutions.add((w, h)) + resolutions.add((h, w)) # portrait / landscape + return sorted(resolutions) + + +def get_valid_depth_color_pairs(): + """ + Generate valid combinations of color and bit depth values. + + Yields tuples of (depth, color) representing valid color mode and bit depth + combinations. Supports grayscale (0), RGB (2, 3, 4, 6) color modes with + appropriate bit depths (1, 2, 4, 8, 16 bits). + + Yields: + tuple: A tuple of (depth, color) where depth is an integer representing + bit depth and color is an integer representing the color mode. + """ + valid_color_depth = { + 0: [1, 2, 4, 8, 16], + 2: [8, 16], + 3: [1, 2, 4, 8], + 4: [8, 16], + 6: [8, 16], + } + for color, depths in valid_color_depth.items(): + for depth in depths: + yield depth, color + + +class PNG: + """ + A class representing a PNG image header (IHDR chunk). + This class encapsulates the metadata from a PNG file's IHDR (Image Header) chunk, + including image dimensions, bit depth, color type, interlacing method, and CRC checksum. + It provides methods for parsing PNG files, packing/unpacking header data, and generating + PNG crc if omitted. + """ + + width: int + height: int + bit_depth: int + color_type: int + crc: int + interlace: int = 0 + + def __init__( + self, + width: int, + height: int, + bit_depth: int, + color_type: int, + crc: int | None = None, + interlace: int = 0, + ): + """Initialize a PNG IHDR (Image Header) chunk. + + Args: + width (int): The width of the image in pixels. + height (int): The height of the image in pixels. + bit_depth (int): The number of bits per sample or per palette index (1, 2, 4, 8, or 16). + color_type (int): The color type of the image (0=Grayscale, 2=RGB, 3=Indexed, + 4=Grayscale+Alpha, 6=RGBA). + crc (int | None, optional): CRC (Cyclic Redundancy Check) checksum for the chunk. + If None, it will be calculated automatically. Defaults + to None. + interlace (int, optional): The interlace method (0=no interlace, 1=Adam7 interlace). + Defaults to 0. + """ + self.width = width + self.height = height + self.bit_depth = bit_depth + self.color_type = color_type + self.crc = ( + crc + or zlib.crc32( + b"IHDR" + + width.to_bytes(4, "big") + + height.to_bytes(4, "big") + + bytes((bit_depth, color_type, 0, 0, interlace)) + ) + & 0xFFFFFFFF + ) + self.interlace = interlace + + @classmethod + def from_packed(cls, packed: int, crc: int | None = None) -> Self: + """Create a PNG instance from a packed integer representation. + + Parse a packed integer containing width, height, bit depth, color type and interlace + bits and return a PNG instance. The CRC may be supplied; if omitted it will be + computed by the constructor. + + Args: + packed (int): Packed integer with fields laid out as described in the `packed` property. + crc (int | None): Optional CRC value for the IHDR chunk. + + Returns: + Self: PNG instance with parsed fields. + """ + interlace = packed & 0b1 + color = (packed >> 1) & 0b111 + depth = (packed >> 4) & 0b111 + height = (packed >> 7) & 0xFFFF + width = (packed >> 23) & 0xFFFF + return cls( + width=width, + height=height, + bit_depth=depth, + color_type=color, + crc=crc, + interlace=interlace, + ) + + @cached_property + def packed(self) -> int: + """Pack IHDR fields into a single integer. + + Returns: + int: Packed integer encoding the IHDR fields. + + Layout: + width : 16 bits + height : 16 bits + depth : 3 bits + color : 3 bits + interlace : 1 bit + """ + return ( + ((self.width & 0xFFFF) << 23) + | ((self.height & 0xFFFF) << 7) + | ((self.bit_depth & 0b111) << 4) + | ((self.color_type & 0b111) << 1) + | (self.interlace & 0b1) + ) + + @classmethod + def from_bytearray(cls, data: bytearray) -> Self: + """Create a PNG image instance from a bytearray containing PNG file data. + + Parses the PNG signature and IHDR (Image Header) chunk to extract image metadata + including dimensions, bit depth, color type, and interlacing information. + + Args: + data (bytearray): A bytearray containing the complete PNG file data, starting + with the PNG signature (8 bytes: \\x89PNG\\r\\n\\x1a\\n). + + Returns: + Self | None: A new instance of the PNG image class with parsed IHDR metadata + (width, height, bit_depth, color_type, crc, interlace) if the data is + valid and contains a proper IHDR chunk. Returns None if the PNG signature + is invalid or the IHDR chunk type is not found. + + Raises: + struct.error: If the data is too short or malformed for unpacking IHDR fields. + IndexError: If the bytearray is shorter than expected for the PNG structure. + """ + + png_signature = b"\x89PNG\r\n\x1a\n" + if data[:8] != png_signature: + raise ValueError("Invalid PNG signature") + offset = 8 # Read IHDR chunk + length = struct.unpack(">I", data[offset : offset + 4])[0] + offset += 4 + chunk_type = data[offset : offset + 4] + offset += 4 + if chunk_type != b"IHDR": + raise ValueError("IHDR chunk not found at expected position") + + ihdr_data = data[offset : offset + length] + offset += length + + crc = struct.unpack(">I", data[offset : offset + 4])[0] + + # Unpack IHDR fields + width, height, depth, color, _, _, interlace = struct.unpack( + ">IIBBBBB", ihdr_data + ) + + return cls( + width=width, + height=height, + bit_depth=depth, + color_type=color, + crc=crc, + interlace=interlace, + ) + + def __repr__(self) -> str: + return ( + f"PNG(width={self.width}, height={self.height}, " + f"bit_depth={self.bit_depth}, color_type={self.color_type}, " + f"interlace={self.interlace}, crc=0x{self.crc:08x})" + ) diff --git a/aperisolve/app.py b/aperisolve/app.py index 5e92077..e81de53 100644 --- a/aperisolve/app.py +++ b/aperisolve/app.py @@ -1,4 +1,4 @@ -"""AperiSolve Flask application.""" +"""Aperi'Solve Flask application.""" import hashlib import json @@ -41,7 +41,7 @@ def too_large(_: Any) -> tuple[Response, int]: @app.errorhandler(404) def not_found(_: Any) -> str: """Error Handler for 404 not found.""" - return render_template("error.html", message="Ressource not found", statuscode=404) + return render_template("error.html", message="Resource not found", statuscode=404) @app.route("/") @@ -134,12 +134,12 @@ def upload_image() -> tuple[Response, int]: first_submission_date=datetime.now(timezone.utc), last_submission_date=datetime.now(timezone.utc), ) - db.session.add(new_img) - db.session.commit() + db.session.add(new_img) # pylint: disable=no-member + db.session.commit() # pylint: disable=no-member sub_img = Image.query.filter_by(hash=img_hash).first() # type: ignore sub_img.upload_count += 1 - db.session.commit() + db.session.commit() # pylint: disable=no-member # Create new Submission entry submission_path.mkdir(parents=True, exist_ok=True) @@ -152,9 +152,9 @@ def upload_image() -> tuple[Response, int]: date=time.time(), image_hash=sub_img.hash, # type: ignore ) - db.session.add(submission) + db.session.add(submission) # pylint: disable=no-member - db.session.commit() # Commit to save the new Image and Submission + db.session.commit() # pylint: disable=no-member # Start the analysis jobs queue.enqueue("aperisolve.workers.analyze_image", submission.hash, job_timeout=300) diff --git a/aperisolve/cleanup.py b/aperisolve/cleanup.py index 130542d..402cd25 100644 --- a/aperisolve/cleanup.py +++ b/aperisolve/cleanup.py @@ -14,7 +14,28 @@ MAX_STORE_TIME = int(getenv("MAX_STORE_TIME", "259200")) # 3 days by default def cleanup_old_entries() -> None: - """Clean up old entries in the database and file system.""" + """ + Clean up old and incomplete entries from the database and file system. + + This function performs two main cleanup operations: + + 1. Submission cleanup: + - Deletes submissions with "pending" or "running" status that have exceeded + the maximum allowed processing time (MAX_PENDING_TIME). + - Deletes completed submissions ("done" status) that have missing or buggy + results (results.json file not found). + + 2. Image cleanup: + - Deletes images older than MAX_STORE_TIME along with all associated + submissions and their result folders. + - Deletes orphaned images (with no submissions) older than MAX_PENDING_TIME + and removes their associated result folders from the file system. + + All database changes are committed at the end of the operation. + + Returns: + None + """ now = time.time() for submission in Submission.query.all(): # type: ignore if ( @@ -22,7 +43,7 @@ def cleanup_old_entries() -> None: and now - submission.date > MAX_PENDING_TIME # type: ignore ): # Processing took too long, delete - db.session.delete(submission) # type: ignore + db.session.delete(submission) # pylint: disable=no-member elif submission.status == "done": # type: ignore # Search for buggy results, delete result_path = ( @@ -30,7 +51,7 @@ def cleanup_old_entries() -> None: ) result_file = result_path / "results.json" if not result_file.exists(): - db.session.delete(submission) # type: ignore + db.session.delete(submission) # pylint: disable=no-member shutil.rmtree(result_path) # Delete "old" images @@ -43,15 +64,15 @@ def cleanup_old_entries() -> None: img_fold = RESULT_FOLDER / img.hash if delay.total_seconds() > MAX_STORE_TIME: for s in img.submissions: - db.session.delete(s) # type: ignore + db.session.delete(s) # pylint: disable=no-member if img_fold.exists(): shutil.rmtree(img_fold) # type: ignore - db.session.delete(img) + db.session.delete(img) # pylint: disable=no-member # Delete Images with missing submission if len(img.submissions) == 0 and delay.total_seconds() > MAX_PENDING_TIME: if img_fold.exists(): shutil.rmtree(img_fold) # type: ignore - db.session.delete(img) + db.session.delete(img) # pylint: disable=no-member - db.session.commit() + db.session.commit() # pylint: disable=no-member diff --git a/aperisolve/config.py b/aperisolve/config.py index fe000d3..117e852 100644 --- a/aperisolve/config.py +++ b/aperisolve/config.py @@ -1,4 +1,4 @@ -"""AperiSolve configuration variables.""" +"""Aperi'Solve configuration variables.""" from pathlib import Path diff --git a/aperisolve/models.py b/aperisolve/models.py index 730bc63..d0f3d7e 100644 --- a/aperisolve/models.py +++ b/aperisolve/models.py @@ -1,11 +1,13 @@ -"""This module defines the database models for the Aperisolve application.""" +"""This module defines the database models for the Aperi'Solve application.""" +import itertools from datetime import datetime, timezone -from decimal import Decimal from flask import Flask from flask_sqlalchemy import SQLAlchemy -from sqlalchemy import Boolean, Column, DateTime, Float, Integer, String +from sqlalchemy import BigInteger, Boolean, Column, DateTime, Float, Integer, String + +from aperisolve.analyzers.utils import PNG, get_resolutions, get_valid_depth_color_pairs db: SQLAlchemy = SQLAlchemy() @@ -13,21 +15,21 @@ db: SQLAlchemy = SQLAlchemy() class Image(db.Model): # type: ignore """Model representing an image file in the database.""" - hash: Column[str] = Column( + hash: Column[String] = Column( String(64), primary_key=True, unique=True, nullable=False ) - file: Column[str] = Column(String(128), unique=True, nullable=False) - size: Column[int] = Column(Integer, nullable=False) - first_submission_date: Column[datetime] = Column( + file: Column[String] = Column(String(128), unique=True, nullable=False) + size: Column[Integer] = Column(Integer, nullable=False) + first_submission_date: Column[DateTime] = Column( DateTime, nullable=False, default=lambda: datetime.now(timezone.utc) ) - last_submission_date: Column[datetime] = Column( + last_submission_date: Column[DateTime] = Column( DateTime, nullable=False, default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc), ) - upload_count: Column[int] = Column(Integer) + upload_count: Column[Integer] = Column(Integer) submissions = db.relationship("Submission", backref="image", lazy=True) @@ -37,14 +39,14 @@ class Submission(db.Model): # type: ignore option. """ - hash: Column[str] = Column( + hash: Column[String] = Column( String(128), primary_key=True, unique=True, nullable=False ) - filename: Column[str] = Column(String(128), nullable=False) - password: Column[str] = Column(String(128)) - deep_analysis: Column[bool] = Column(Boolean, default=False) - status: Column[str] = Column(String(20), default="pending") - date: Column[Decimal] = Column( + filename: Column[String] = Column(String(128), nullable=False) + password: Column[String] = Column(String(128)) + deep_analysis: Column[Boolean] = Column(Boolean, default=False) + status: Column[String] = Column(String(20), default="pending") + date: Column[Float] = Column( Float, nullable=False, default=lambda: datetime.now(timezone.utc) ) @@ -52,10 +54,84 @@ class Submission(db.Model): # type: ignore image_hash = Column(String, db.ForeignKey("image.hash"), nullable=False) +class IHDR(db.Model): # type: ignore + """IHDR CRC lookup table""" + + iid: Column[Integer] = Column(Integer, primary_key=True, autoincrement=True) + crc: Column[BigInteger] = Column(BigInteger, nullable=False) + packed: Column[BigInteger] = Column(BigInteger, nullable=False) + + +def create_crc_db() -> None: + """ + Creates a database of common PNG IHDR (Image Header) entries by generating + combinations of standard PNG parameters such as resolutions, bit depths, + color types, and interlace methods. The function checks for existing entries + in the database to avoid duplicates and inserts new entries in batches of + 5000, committing the changes to the database periodically. + + Returns: + None + """ + # Standard PNG IHDR parameters + resolutions = get_resolutions() + bit_color_pairs = get_valid_depth_color_pairs() + interlace_methods = [0, 1] + + combinations = itertools.product( + resolutions, + bit_color_pairs, + interlace_methods, + ) + + count = 0 + + with db.session.no_autoflush: # pylint: disable=no-member + for (w, h), (bd, ct), inter in combinations: + p = PNG(width=w, height=h, bit_depth=bd, color_type=ct, interlace=inter) + + exists = db.session.execute( # pylint: disable=no-member + db.select(IHDR.iid).where(IHDR.packed == p.packed) + ).first() + + if exists: # Skip if already present + continue + + db.session.add(IHDR(packed=p.packed, crc=p.crc)) # pylint: disable=no-member + count += 1 + + if count % 5000 == 0: + print(f"Inserted {count} common IHDR entries...") + db.session.commit() + + db.session.commit() # pylint: disable=no-member + print(f"Precomputed {count} common IHDR entries.") + + def init_db(app: Flask) -> None: - """Initialize the database with the given Flask app.""" + """ + Initialize the database by creating tables and populating lookup data. + + This function creates the database schema if it doesn't already exist, + and populates the CRC lookup table for PNG IHDR validation. + + Args: + app (Flask): The Flask application instance used to establish the application context + for database operations. + + Returns: + None + + Side Effects: + - Creates all database tables if the "image" table doesn't exist + - Populates the IHDR CRC lookup table with initial data + - Prints status messages to console during initialization + """ with app.app_context(): if not db.engine.dialect.has_table(db.engine.connect(), "image"): print("Creating database...") db.create_all() - print("Database created successfully.") + print("Database structure created successfully.") + print("Filling IHDR lookup table...") + create_crc_db() + print("IHDR table filled successfully.") diff --git a/aperisolve/static/js/aperisolve.js b/aperisolve/static/js/aperisolve.js index 6e98b57..2867666 100644 --- a/aperisolve/static/js/aperisolve.js +++ b/aperisolve/static/js/aperisolve.js @@ -279,6 +279,12 @@ function parseResult(result) { )}'/>`; } + if ("png_images" in result[tool]) { + for (const image of result[tool]["png_images"]) { + analyzer.innerHTML += `
`; + } + } + // Parse text output diff --git a/aperisolve/workers.py b/aperisolve/workers.py index 956c328..acb6b8e 100644 --- a/aperisolve/workers.py +++ b/aperisolve/workers.py @@ -8,19 +8,43 @@ from .analyzers.binwalk import analyze_binwalk from .analyzers.decomposer import analyze_decomposer from .analyzers.exiftool import analyze_exiftool from .analyzers.foremost import analyze_foremost +from .analyzers.image_resize import analyze_image_resize from .analyzers.outguess import analyze_outguess +from .analyzers.pngcheck import analyze_pngcheck from .analyzers.steghide import analyze_steghide from .analyzers.strings import analyze_strings from .analyzers.zsteg import analyze_zsteg -from .analyzers.pngcheck import analyze_pngcheck -from .analyzers.image_resize import analyze_image_resize from .app import app, db from .config import RESULT_FOLDER from .models import Image, Submission def analyze_image(submission_hash: str) -> None: - """Analyze an image submission in separate threads.""" + """ + Analyze an image submission by running multiple analysis tools concurrently. + + This function retrieves a submission and its associated image from the database, + then executes various image analysis tools in parallel threads. The analysis tools + include binwalk, decomposer, exiftool, foremost, strings, pngcheck, steghide, + zsteg, and image resize. If deep analysis is enabled, additional tools like + outguess are included. + + Args: + submission_hash (str): The unique hash identifier of the submission to analyze. + + Returns: + None + + Raises: + Implicitly catches and handles exceptions during analysis, setting submission + status to "error" if an exception occurs. + + Side Effects: + - Queries the database for the submission and associated image + - Updates submission status in the database ("running" -> "completed" or "error") + - Creates result directories and generates analysis output files + - Modifies the database session and commits changes + """ with app.app_context(): submission: Submission = Submission.query.get(submission_hash) # type: ignore image = Image.query.get_or_404(submission.image_hash) # type: ignore @@ -29,7 +53,7 @@ def analyze_image(submission_hash: str) -> None: return submission.status = "running" # type: ignore - db.session.commit() + db.session.commit() # pylint: disable=no-member try: img_path: Path = Path(image.file) @@ -77,4 +101,4 @@ def analyze_image(submission_hash: str) -> None: except Exception: submission.status = "error" finally: - db.session.commit() + db.session.commit() # pylint: disable=no-member diff --git a/examples/size_change_png.png b/examples/size_change_png.png new file mode 100644 index 0000000..064c96a Binary files /dev/null and b/examples/size_change_png.png differ